diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml index e7ed27a6..f60f73cc 100644 --- a/.github/workflows/build-pr.yml +++ b/.github/workflows/build-pr.yml @@ -20,4 +20,16 @@ jobs: distribution: temurin java-version: 25 - name: Build on ${{ matrix.os }} - run: ./gradlew clean build \ No newline at end of file + run: ./gradlew clean build + # Gradle's console summary names the failing test and the exception type, but not the + # message, the path or the stack trace. For a failure that only happens on one operating + # system, that is the difference between reading the cause and guessing at it. + - name: Upload test reports + if: failure() + uses: actions/upload-artifact@v4 + with: + name: test-reports-${{ matrix.os }} + path: | + build/reports/tests/ + build/test-results/ + retention-days: 7 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1b71ca16..e25b7284 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,7 @@ bin/ # OS-spezifisch .DS_Store -Thumbs.db \ No newline at end of file +Thumbs.db + +# Serena (MCP code navigation) +.serena/ diff --git a/STATUS.md b/STATUS.md new file mode 100644 index 00000000..da08ff6b --- /dev/null +++ b/STATUS.md @@ -0,0 +1,619 @@ +# Status — Anvil chunk loader and light engine + +Branch `feat/aves-anvil-chunk-loader` · PR [#91](https://github.com/OneLiteFeatherNET/Aves/pull/91) +against `develop` · 34 commits · **575 tests** · `clean build` green. + +Everything below is experimental and opt-in. `AbstractMapProvider` still uses the loader Minestom +ships with unless a `ChunkLoaderFactory` is passed explicitly, so no existing consumer changes +behaviour by upgrading. + +--- + +## What this is and why + +Aves is OneLiteFeather's utility library for Minestom servers. This branch adds two things it did +not have: + +1. **An Anvil chunk loader** that replaces `net.minestom.server.instance.anvil.AnvilLoader`. The + goal was a loader that is genuinely parallel, that does not silently lose data, and that stays + maintainable — developed test-first, on Java 25, using Adventure NBT and JetBrains annotations. +2. **A light engine**, because once chunks are loaded, lighting is the next thing a server pays for. + +The motivating observation for the loader: Minestom's `AnvilLoader` reports +`supportsParallelLoading() == true`, but its `RegionFile` serialises reading, decompression **and** +NBT parsing through a single `ReentrantLock`. The parallelism is largely nominal. The gain is not in +starting more threads but in moving the CPU work out of the lock — which is what the three-stage +pipeline below does, and what the measurements confirmed. + +## Environment + +| | | +| --- | --- | +| Java | 25 (toolchain and `release`), no `--enable-preview` anywhere | +| Minestom | `2026.06.20-26.1.2`, `compileOnly` | +| Adventure | `5.1.1`, `adventure-nbt` used directly — Minestom speaks `CompoundBinaryTag` natively, so there is no conversion layer | +| Annotations | `org.jetbrains:annotations:26.1.0` | +| Tests | JUnit 6.1.0, Cyano `0.6.2` (`MicrotusExtension`) for anything needing a server | +| Benchmarks | JMH 1.37 via `me.champeau.jmh` 0.7.3, own `src/jmh/java` source set | +| Build | Gradle 9.6.1 | + +`adventure-nbt`, `jetbrains-annotations` and `fastutil` were added to the version catalog by this +branch. The first two were only reaching the classpath transitively through `compileOnly(minestom)`, +so any direct use compiled by coincidence. `fastutil` is `runtime` scope in Minestom's POM and is +needed only by the comparison benchmark. + +## Working on this + +```bash +./gradlew clean build # compile, javadoc, tests — javadoc failures break the build +./gradlew test --tests "*Anvil*" # a subset +./gradlew jmhJar # build the benchmarks (they never run during build) +java -jar build/libs/aves-*-jmh.jar ScalingBenchmark -f 1 -wi 2 -i 3 +``` + +Two things that will otherwise cost an hour: + +- **Do not run two Gradle builds in the same checkout at once.** They corrupt `build/test-results` + and surface as `EOFException`, `NoClassDefFoundError` or missing `jacoco/test.exec` — failures + that look like real test breakage. `rm -rf build` and rerun. +- **JMH allows one instance at a time.** A crashed run leaves `/tmp/jmh.lock` behind and every later + run fails with *"Another JMH instance might be running"*. Delete the file. + +## Conventions + +Match these; the build enforces some of them. + +- **Javadoc on every class and method**, with `@param` / `@return` / `@throws`. `withJavadocJar()` + means an incomplete comment fails CI. Class comments explain *why*, not *what*, and carry + `@author` / `@version` / `@since`. +- **Never write `@NotNull`.** Packages carry `@NotNullByDefault` in `package-info.java`; only + `@Nullable`, `@Contract` and `@UnmodifiableView` appear explicitly. +- **Test-first, strictly.** Every type here was built by writing a failing test, confirming it fails + for the right reason, then implementing. Several bugs in this branch were found precisely because + a test was written before the code. +- Tests are package-private, named `test`, use plain JUnit assertions, and avoid + `@Nested`. Anything needing a server uses `@ExtendWith(MicrotusExtension.class)`. +- Commits follow conventional commits, scoped `(anvil)`, `(light)`, `(map)`. + +## Facts that cost real effort to establish + +Verified against the sources or by running probe code. Knowing these prevents repeating the work. + +**Minestom's loader interface** +- `ChunkLoader#loadChunk` is **synchronous** — it returns `@Nullable Chunk`, not a future. + Parallelism happens because Minestom starts a virtual thread per chunk when + `supportsParallelLoading()` is true. A loader must be thread-safe, not asynchronous. +- The default `saveChunks` starts **one virtual thread per chunk**, unbounded, and its `catch` + branch never deregisters from the `Phaser`, so one exception hangs it forever. Override it. +- `unloadChunk` is documented as arriving for chunks the loader never loaded, which makes reference + counting on it unreliable. +- `setChunkLoader` does **not** call `loadInstance` — only the constructor does. `AbstractMapProvider` + sets the loader afterwards, so `level.dat` is never read there. Pre-existing, not introduced here. + +**What can and cannot be replaced** +- `Palette` is `sealed ... permits PaletteImpl`. A foreign implementation is a hard compiler error, + and `Section` is a record holding that exact type. Verified with javac and at runtime. +- `Light` is **not** sealed, and `Section`'s canonical constructor is public — a custom light + implementation compiles and runs end to end. But `Section.clone()` calls `Light.sky()` / `Light.block()` + outright, so a custom implementation is silently replaced on copy. +- `Instance` and `InstanceContainer` are not sealed either, but four `instanceof InstanceContainer` + sites in Minestom make a foreign instance silently take a different path. + +**Claims about other light engines** + +Established by reading the sources, not by measuring. Each of these was taken for a promising lead +first and only stopped being one after it was checked. + +- **The main advantage attributed to Starlight is already here.** Aves' BFS pushes a level onto the + neighbours of a cell instead of pulling each cell from all six of its own neighbours, which is the + difference Spottedleaf names as the reason Starlight beats vanilla. This is a structural property + of both implementations, not a measured figure. +- **Starlight has no "extended nibble arrays with a border".** `SWMRNibbleArray.ARRAY_SIZE` is + 2048 bytes, identical to vanilla's `DataLayer`. What does exist is the flat `sectionCache` / + `nibbleCache` over 5×5 sections — that is a flat `byte[]` for the column, not a border. +- **Starlight's data-holding gain does not transfer.** It comes from vanilla keeping light in a + `Long2ObjectOpenHashMap` and cloning it per tick. Aves never had that structure. +- **The quoted 12× / 28× / 37× figures do not apply.** They compare Minecraft 1.16–1.19 against the + old vanilla engine. Spottedleaf withdrew the chunk-generation comparison himself and states for + 1.20+ only "Vanilla is still 2x slower, but it is fast enough". +- **Phosphor's optimisations are vanilla-specific bar one.** The transferable part is the block-state + opacity cache, which is what `SectionOpacity` already is. +- **There is no scientific literature on this problem.** No peer-reviewed work on discrete + Minecraft-style flood-fill light propagation exists; the reference text is a blog post (Ben Arnold, + Seed of Andromeda). Voxel cone tracing and VXGI solve continuous radiance and do not transfer. + +**Library traps** +- `adventure-nbt` 5.1.1: the iterators of `LongArrayBinaryTag`, `IntArrayBinaryTag` and + `ByteArrayBinaryTag` **skip the last element** (`index < length - 1`). A for-each over packed block + data corrupts every chunk. Use `size()` + `get(i)`. `NbtReadsTest` documents this as a live check. +- `BinaryTagIO.reader()` caps at 131 082 bytes, far too small for chunk NBT. Use `unlimitedReader()`. +- Every `CompoundBinaryTag` getter silently returns a default for a missing or mistyped key, which + turns a broken region file into an empty chunk. `NbtReads` exists to make that an error. +- `Block.fromStateId` indexes an array **without a bounds check** and throws for an unknown id + instead of returning null. + +**Test environment** +- `MinecraftServer.getExceptionManager()` throws before `MinecraftServer.init()`. Anything resolving + a registry in a constructor becomes untestable — this is why the biome resolver is lazy. +- Cyano's exception handler turns a reported exception into a **test failure**. Code that reports to + the `ExceptionManager` cannot be asserted on by exception type in tests. + +**Java 25** +- `StructuredTaskScope` (JEP 505) and `StableValue` (JEP 502) are still **preview** and therefore + unusable in a published library — preview class files only run on the exact JDK they were built + with, and would force `--enable-preview` on every consumer. Concurrency here uses + `Executors.newVirtualThreadPerTaskExecutor()`, `Semaphore` and `Phaser`. +- **The Vector API (JEP 508) is the same trap.** It is the tenth incubator round: without + `--add-modules jdk.incubator.vector` `javac` already refuses, with it the runtime prints a warning + that cannot be suppressed, and the JAR specification has no `Add-Modules` attribute to carry the + flag. Every consumer of the library would have to set a JVM flag, which rules it out regardless of + what it might buy. +- Scoped Values, record patterns, sealed interfaces, FFM and stream gatherers are final and usable. +- File I/O does **not** unmount a virtual thread from its carrier (JEP 444), so unbounded virtual + threads over file work do not scale — bound them. + +## Decisions that shape everything else + +These were explicit calls, not defaults. Changing one means revisiting the work that followed it. + +| Decision | Choice | Why | +| --- | --- | --- | +| Format coverage | Core compression plus external `.mcc`, **no** LZ4, no corruption recovery | Covers real worlds without an extra dependency; Minestom fails hard on oversized chunks, which this does not | +| Integration | Opt-in via `ChunkLoaderFactory` | No breaking change; existing providers behave exactly as before | +| Own palette | **Not built** — codec-internal representation only | `Palette` is sealed, and it is 4.5 % of the load path anyway | +| Own `InstanceContainer` | **Not built** | Compiles, but four `instanceof` sites break silently and the tick parallelism lives elsewhere | +| Light `Light` implementation | **Not built** — results handed over via `Light#set` | Avoids the `@ApiStatus.Internal` calculation methods and the `Section.clone()` trap | +| Read failure | Throws, never returns `null` | `null` means "absent", so the server regenerates and overwrites real data on the next save | +| Compression level | 2, not the platform default 6 | 1.83× faster for ~3 % more bytes; compression is 63 % of a save | +| Reader safety in `RegionFile` | Per-entry seqlock | A `ReadWriteLock` would block every reader of a region for the length of a payload write, which is the one thing this loader gains over Minestom's; deferred free brings the contention back through a shared counter and grows the files | +| Region handle lifetime | Usage count, not a retry | A retry only narrows the window and has to be repeated at every call site; counting accesses removes the case instead. The cost is that the open-handle cap now bounds the cache rather than the descriptors | +| Use after `close()` | Throws | Ignoring it loses data during shutdown, and waiting is impossible — Minestom owns the load tasks, so there is nothing to wait on | + +## Where things live + +``` +src/main/java/net/theevilreaper/aves/ + instance/anvil/ RegionConstants, SectorAllocator, BitPacker, ChunkCompression, + RegionFile, NbtReads, PaletteData, PaletteEntryResolver, SectionCodec, + BlockPaletteResolver, BiomePaletteResolver, AnvilDiagnostics, + AvesAnvilLoader, AnvilChunkException + instance/light/ LightNibbles, BlockFace, BlockLightSource, SectionOpacity, + LightPropagator, ChunkLightPropagator, ChunkLightState, + ChunkLightService, MinestomBlockLightSource + map/provider/ ChunkLoaderFactory (+ registerInstance overload in AbstractMapProvider) + +src/test/java/... mirrors the above; *ConcurrencyTest are the stress tests, + LightEngineEquivalenceTest pins the byte identity with Minestom +src/jmh/java/ benchmarks; LightEngineComparisonBenchmark and + LightEngineStageBenchmark live in net.minestom.server.instance.light + because the methods they measure are package-private there +``` + +Reading order for someone new: `RegionFile` (the byte container), then `AvesAnvilLoader` +(the three stages), then `SectionOpacity` and `ChunkLightPropagator` for the light side. + +## Charts + +Published from the measurements in this branch. They are snapshots, not live views — re-run the +benchmarks before trusting them after a change. + +| Chart | Shows | +| --- | --- | +| [Scaling and comparison](https://claude.ai/code/artifact/38131b5a-42f8-43c6-a843-f845802d78ae) | 1 to 256 sections, and the head-to-head against Minestom. **The head-to-head half predates `69381af`** and shows the factors from before the opacity table was rewritten | +| [Optimisation](https://claude.ai/code/artifact/a11c1e46-7310-40ed-84e5-0c4d650cbcc1) | Where save time goes, the compression trade-off, the uniform-section fast paths | +| [Vanilla · Minestom · Aves](https://claude.ai/code/artifact/9d3b6d0d-ced3-4f0b-b675-6fb1640f262f) | 22 behaviours scored against the format reference | +| [Concurrency defects](https://claude.ai/code/artifact/9b11a843-8db5-4495-8a95-b0423df28304) | The five races, their failure rates before the fix, and what the fix costs | + +--- + +## What is in the branch + +| Package | Types | Tests | What it does | +| --- | ---: | ---: | --- | +| `instance.anvil` | 14 | 13 classes | Reads and writes Anvil region files, replacing `AnvilLoader` | +| `instance.light` | 9 | 13 classes | Computes block light and sky light for a chunk | +| `map.provider` | +1 | 1 class | `ChunkLoaderFactory`, the opt-in seam | +| `src/jmh` | 23 files | — | Benchmarks, in their own source set | + +### Anvil loader + +Three stages, so the expensive work never happens under a lock: the chunk state is copied under its +read lock, the conversion to compressed bytes runs lock-free, and only the transfer into the region +file is guarded. `saveChunks` is grouped per region and bounded by a semaphore rather than starting +one virtual thread per chunk. + +Region files use positional `FileChannel` operations, so reads of different chunks proceed in +parallel, and a per-entry seqlock keeps a reader from being handed a sector that was recycled while +it read. Every access registers itself on the handle; a file leaves the cache when the last chunk +this loader read from it is unloaded, and is closed by whichever thread finishes with it last. The +cap on open handles is a backstop on the cache, not on descriptors. + +### Light engine + +Block light and sky light, across section borders, across chunk borders, and incrementally after a +single block changed. The algorithm has no Minestom dependency — the registry sits behind +`BlockLightSource`, the same separation the Anvil codec uses for palettes — and results are handed to +a chunk through `Light#set`, which is the stable part of that interface rather than its internal +calculation methods. + +One `ChunkLightService` serves any number of threads, because it keeps no state between calls. That +is a property worth stating rather than assuming: the working buffers live in a propagator built per +call, and handing several threads one propagator is what made the engine produce silently wrong +light before. + +--- + +## Measured + +All figures from one machine that was **not idle**. Ratios are meaningful, absolute microseconds +carry a wide error. Reproduce with `./gradlew jmhJar` and the benchmark names below — except for the +parts of *Where the time goes in the light path* that are still marked as coming from a standalone +rebuild. + +### Where the time goes when saving a chunk + +`ChunkSaveStageBenchmark`, 24 sections, 200 block states: + +| Stage | Time | Lock held | +| --- | ---: | --- | +| Snapshot | 64 µs | chunk read lock | +| Codec, without compression | 1 356 µs | none | +| zlib compression | 2 701 µs | none | +| Transfer | 17 µs | region lock | + +**About 97 % of a save runs outside any lock**, and compression is 63 % of the whole operation. This +is the measurement that turned the design claim into a number, and it is what made compression the +optimisation target. + +### Against the engine Minestom ships with + +`LightEngineComparisonBenchmark`, one section, `-f 1 -wi 5 -i 10`, µs/op. Both engines run to a +**byte-identical** result over all 54 scenarios; since `69381af` that is checked by +`LightEngineEquivalenceTest` on every build and again by the benchmark before each trial, rather +than being asserted from a one-off comparison. + +Measured before and after `69381af` in one session on the same machine, with Minestom as the +control: + +| Sources | Solid | Aves before | Aves after | Minestom | Before | After | +| ---: | ---: | ---: | ---: | ---: | --- | --- | +| 1 | 0 % | 74.2 ± 1.9 | 44.5 ± 0.6 | 49.4 ± 1.3 | 1.42× slower | 1.11× faster | +| 1 | 30 % | 61.8 ± 1.6 | 39.3 ± 0.8 | 62.0 ± 2.0 | 1.03× slower | 1.58× faster | +| 8 | 0 % | 137.7 ± 7.4 | 98.3 ± 2.4 | 121.1 ± 5.5 | 1.18× slower | 1.23× faster | +| 8 | 30 % | 144.7 ± 1.8 | 119.3 ± 3.5 | 204.2 ± 3.7 | 1.37× faster | 1.71× faster | +| 64 | 0 % | 135.9 ± 2.8 | 109.2 ± 1.6 | 126.5 ± 5.6 | 1.08× slower | 1.16× faster | +| 64 | 30 % | 152.7 ± 1.8 | 122.6 ± 1.3 | 206.6 ± 4.2 | 1.37× faster | 1.68× faster | + +Aves is now ahead in all six scenarios instead of two, and the lead on solid blocks grew rather than +being traded for the empty rows. An independent re-run confirms direction and order of magnitude, +not the third digit. This is a result under the conditions named at the top of this section — one +section, one machine that was not idle, sources of equal brightness — and not a general statement +about either engine. + +The earlier reading of the pattern was wrong and is worth recording. It said an empty section favours +Minestom because our opacity table is built unconditionally, and a section with solid blocks favours +us because the table is then read many times. The table is still built unconditionally; it now costs +a quarter. The stage breakdown below also shows that **Aves' search was already the faster of the +two before the change** — 33.9 µs against Minestom's 53.9. The entire deficit came from the +preparation, never from the algorithm. + +### With sources of mixed brightness + +`LightEngineComparisonBenchmark` gained an `emissionMix` parameter in `0e8fbb5`. `MIXED` cycles the +sources through glowstone, lantern, torch, redstone torch and magma block, which the registry gives +15, 15, 14, 7 and 3; positions are drawn identically to `UNIFORM`, so the levels are the only +difference. µs/op: + +| Sources | Solid | Aves | Minestom | +| ---: | ---: | ---: | ---: | +| 8 | 0 % | 118.97 ± 8.89 | 126.54 ± 9.55 | +| 8 | 30 % | 116.50 ± 7.63 | 201.46 ± 16.83 | +| 64 | 0 % | 150.42 ± 32.73 | 162.20 ± 3.11 | +| 64 | 30 % | 149.42 ± 12.64 | 252.26 ± 9.28 | + +Mixed brightness costs Aves **about 33 %** against `UNIFORM` — 64 sources at 0 % solid go from 112.7 +to 150.4 µs — and the lead in that row falls from 1.30× to 1.06×. A position is queued more than +once when sources of different brightness reach it, which is exactly the case a bucket queue is for; +the research predicts −32 to −36 % there. The benchmark can now see it, which is what the decision +under *Investigated and deliberately not built* was waiting for. + +### Scaling by world height + +`ScalingBenchmark`, 1 to 256 sections: + +- **Block light is linear** across the whole range — 33 µs per section at 1 section and at 256. +- **Sky light is not.** Cost per section rises from 84 µs to 104 µs past roughly 64 sections. +- A least-squares fit over the vanilla range (≤ 24 sections) predicts 21 423 µs of sky light at 256 + sections. The measured value is 26 597 µs — **the forecast understates it by 19.5 %**. For block + light the same method lands within 1.8 %. + +Measuring the exotic sizes rather than extrapolating from common ones is the only reason this is +known. The cause of the sky-light curve is named below: seeding queues every open cell. + +### Where the time goes in the light path + +This section began as a standalone rebuild of the same call structure, run outside the project — not +JMH and not the real code. The part of it that mattered most has since been measured for real: +`69381af` added `LightEngineStageBenchmark`, which times the stages of both engines inside the +project. Where a rebuild estimate has been replaced by a JMH figure that is said below; the rest is +still the rebuild and still carries only its ratios. + +**Measured, `LightEngineStageBenchmark`, 1 source, 0 % solid, µs:** + +| Stage | Before `69381af` | After | +| --- | ---: | ---: | +| `opacity` — build the table | 31.33 | **8.07** | +| `readStates` | 7.70 | 7.23 | +| `propagate` — the search itself | 33.85 | 31.41 | +| `collect` | 0.24 | 0.23 | +| Total | 77.1 | 46.3 | + +Allocation while building the table: 74 040 → 8 664 bytes per call. + +Two things fall out of this. The rebuild's estimate for `SectionOpacity.of` — 29.1 µs against 5–7 µs +for a table without boxing — was close enough on both ends; the real path lands at 8.07 µs with a +local linear-probing table over the raw state id. And **the search was never the problem**: at 33.9 µs +`propagate` was already faster than Minestom's 53.9 µs before any of this. The whole deficit against +Minestom sat in the preparation. + +The mechanism the rebuild identified was correct. The lambda handed to `computeIfAbsent` captures the +`BlockLightSource`, so a fresh instance is created on every loop iteration, and escape analysis does +not remove it because `computeIfAbsent` is too large to inline — 4096 objects per section, roughly +1.8 MB per chunk column. That is also where the ±19.1 spread against Minestom's ±2.7 came from. + +The rest is still **the rebuild: not JMH, not the real code**, run on Temurin 25, best of seven. The +ratios between variants carry; the absolute microseconds are coarser than everything else in this +section. Ordered by the size of the effect, with what has since been built marked as such: + +- **Done in `69381af`: the opacity table without a per-block allocation.** The rebuild put this at + 20–45 % of the path; the stage benchmark above is the real figure. +- **Done in `69381af`: `collect()` as one linear nibble pack** instead of writing position by + position through `LightNibbles.set` and cloning afterwards. The rebuild put it at 9.6 → 1.2 µs in + the non-uniform case; `LightNibbles.ofLevels` now packs two neighbours at a time and range-checks + once at the end. +- **Open: seeding sky light from a heightmap** instead of queueing every open column cell: 79.3 → + 55.1 µs, and 81 000 → 19 000 queued positions. Byte identity against the current seeding was + verified over 240 randomly generated worlds, zero differing cells. This is what the non-linear + sky-light scaling above is made of. +- **Open, and deliberately so: the seed pass is redundant.** `seed` walks all 4096 positions only to + find the emitters, which `of` already visits: 3.6 µs, plus 2.0 µs for the second `byte[4096]` that + then becomes unnecessary. Left out of `69381af` because writing the emitters during the table build + changes the API of `SectionOpacity` and both propagators for a gain of that size. +- **Open: column opacity as one flat `byte[]`** instead of `List.get(y >> 4)` plus a virtual call: + −26 % on searching a whole column. +- **Open: skipping the direction an entry arrived from**: −7 to −16 %. **Testing the level before the + opacity**: −6 %. +- **A bucket queue (Dial)** is 5–7 % *slower* at equal source brightness and 32–36 % faster at mixed + brightness. The benchmark now produces the mixed case — see *With sources of mixed brightness* + above, where mixed sources cost about 33 %. +- **`ChunkLightState` allocates about 980 KB of buffers per instance**, and `calculateWithNeighbours` + builds nine of them — roughly 28 MB of garbage per call. Derived from the buffer sizes, not + measured with an allocation profiler. + +### Optimisations these numbers produced + +| Change | Effect | +| --- | --- | +| zlib level 2 instead of the platform default 6 | 1.83× faster compression, ~3 % larger files | +| Fast path for uniform sections, palette encode | 27.9 µs → 0.54 µs (**51×**) | +| Fast path for uniform sections, opacity table | 40.8 µs → 0.54 µs (**76×**), and no arrays allocated | +| Linear-probing opacity table over the raw state id, no boxing | 31.33 µs → 8.07 µs, 74 040 → 8 664 bytes per call | +| `LightNibbles.ofLevels` instead of 4096 calls to `set` | `collect` 0.24 µs → 0.23 µs in the stage benchmark; the rebuild had it at 9.6 → 1.2 µs for the non-uniform case | + +--- + +## Known deviations from vanilla + +Vanilla defines the Anvil format, so these are gaps in this implementation, not preferences: + +| Gap | Consequence | +| --- | --- | +| Heightmaps are neither written nor restored | Minestom at least restores them on load | +| Unknown chunk-level tags are dropped | `structures`, `block_ticks`, `fluid_ticks` and others are lost on save | +| `entities/` and `poi/` are ignored | Saving a vanilla world produces inconsistent world data | +| `level.dat` is not handled | `loadInstance` / `saveInstance` are not overridden | +| No LZ4 (type 4) or custom (type 127) compression | A world written with `region-file-compression=lz4` cannot be read | +| No corruption recovery | A damaged header makes the whole region unreadable | +| An unknown block becomes air | Better than discarding the chunk, but not what a data fixer does | + +--- + +## Defects found and fixed + +### In this code + +- **Block entities were stored at chunk-local coordinates.** The format specifies world coordinates. + The round trip through this loader worked anyway because `Chunk#setBlock` masks them, so only a + test that read the stored NBT directly could catch it. Files were not interchangeable with vanilla. +- **Block handlers were lost on load.** The `id` tag was written on save but discarded on load. +- **The propagation queue could overflow.** It was sized on the assumption that a position is queued + at most once, which is false when sources of different brightness reach the same area. +- **`AvesAnvilLoader` could lose a chunk.** A region file could be evicted between obtaining the + handle and writing to it. +- **The name cap in `AnvilDiagnostics` was a check-then-act**, so racing threads could exceed it. +- **The biome registry was resolved eagerly**, which made a loader impossible to construct before + `MinecraftServer.init`. +- **The byte identity against Minestom was never checked by anything.** This file and the documents + stated "54 scenarios, byte-identical, zero differing cells" as an established fact. It rested on an + ad-hoc comparison run once by hand: there was no test, and the benchmark did not verify it either, + although the documentation said it did. Two agents found this independently at their own end of the + code. `LightEngineEquivalenceTest` now runs the 54 scenarios on every build, and the benchmark + checks the 2048 bytes of both engines before each trial. A number cited throughout was hanging on + nothing, which is the part worth remembering — not that it turned out to hold. + +### Five races, all of which would have failed silently + +Found by taking one known defect and searching the rest of the code for the same shape. The search +turned up no second instance of that exact shape, but four races of other kinds — which is the +reason it was worth doing. Numbers below are from the red run of each test; charted +[here](https://claude.ai/code/artifact/9b11a843-8db5-4495-8a95-b0423df28304). + +- **`ChunkLightService` shared its scratch buffers.** It kept a `ChunkLightPropagator` in a field, + so two threads sharing one service shared its `levels` and `queue` arrays. A probe found wrong + light in ~99 % of concurrent calls. `ChunkLightState` had built one per call all along, which is + why `calculateWithNeighbours` was never affected. +- **`RegionFile` recycled sectors while readers were still in them.** `readRaw` took the location + without a lock; a concurrent `writeRaw` freed the old range, which the allocator handed straight + back out. Readers observed filler markers where their own payload belonged, sometimes the whole + payload from offset 0. Now a per-entry seqlock: an odd counter means "in progress", and after four + attempts the reader falls back to the lock so a chunk written in a loop cannot starve it. +- **`.mcc` files were written and deleted outside the header lock.** 371 `NoSuchFileException` and + ~60 half-read files in 54 679 reads. The payload now goes to a staging file and is moved into + place with `ATOMIC_MOVE` under the lock, so the bytes stay outside it while the header and the + file can never disagree. +- **Eviction closed a channel under a running reader.** Chunk tracking starts only *after* decoding, + so a handle could be closed mid-read: 80 of 480 loads failed with `openRegionLimit = 1`, 15 of 480 + through the unload path, with `ClosedChannelException` thrown from inside `FileChannelImpl.read`. + Handles now carry a usage count; removing from the cache and closing are separate, and the last + user closes. +- **`closed` was set but never read.** `loadChunk`, `saveChunk` and `region()` ignored it, so a load + still running at shutdown opened fresh handles into the map `close()` had just cleared — a + descriptor leak, and writes into a world already considered closed. They now throw + `IllegalStateException`, which is a lifecycle error of the caller rather than a data error. + +The reason all five mattered is that none of them announced itself: the light path clears the +section's update flag, so the server never recomputes what two threads corrupted, and a read failure +that returns `null` makes Minestom regenerate the chunk and overwrite the real data on the next save. + +### A sixth, on Windows only, and older than the five + +`RegionFile` wrote and deleted the external `.mcc` file of an oversized chunk in a way that let a +concurrent reader block it. The external file is the one place where the lock-free reads meet a name +in the file system rather than a range inside the region file, and a name is not a POSIX concept. +Under POSIX a deletion detaches the name at once and keeps the unnamed file alive for every open +handle, so nothing is noticed. Windows leaves the name in the directory and only marks the file +*delete-pending*: as long as one reader holds it open, every later open of that name and every move +onto it is denied. An inline writer therefore poisoned the name for the writer that wanted to put a +new external file there. + +Fixed in `78e196c`: the file is renamed onto a private name with `ATOMIC_MOVE` and only then deleted, +because a rename detaches the name immediately on both systems. What Windows can still deny briefly +on its own — a handle being torn down, a virus scanner holding the file — is retried for a bounded +time. + +**This one is older than the concurrency fixes of this week.** It was already in the last green +commit; there simply was no test that exercised it. The loader was therefore broken on Windows as +soon as an oversized chunk is read while it is being saved. Only the CI runner could show it — on +Linux it is not reproducible, and the platform semantics are what the fix had to be reasoned from. + +That is also why `.github/workflows/build-pr.yml` now uploads the test reports as an artifact when a +build fails (`bec8b67`). Gradle's console summary names the test class and the exception type, but +neither the message nor a path nor a stack trace, and on a platform-specific failure that is the +difference between reading the cause and guessing it. + +### In Minestom, avoided here + +Length field written as `5 + N` instead of `1 + N`; `status` in lower case where the game writes +`Status`; the return value of `read` ignored; an unknown block turning into an NPE that discards the +whole chunk; block entities dropped in uniform sections; the read failure path returning `null`, +which makes the server regenerate the chunk and overwrite the real data on the next save. + +--- + +## Open + +Ordered by consequence, not by effort. + +### 1. Exception hierarchy + +Design complete in [`docs/research/exception-hierarchy.md`](docs/research/exception-hierarchy.md), +six types, not implemented. **One open decision:** whether the checked root extends `IOException`. +Extending it keeps roughly 40 signatures and 14 test assertions untouched; not extending it stops +every existing `catch (IOException)` from silently swallowing the new types. Both arguments hold — +this needs a call, not more analysis. + +### 2. `calculateWithNeighbours` is last-writer-wins across chunks + +One service may now serve any number of threads, which is what the light fix established. What it +does **not** establish is two threads lighting *overlapping neighbourhoods*: each reads the block +states of all nine chunks separately, then both write into the same sections. Neither corrupts +memory — every write holds the chunk's write lock — but the later writer wins on the basis of a read +that may already be stale, which shows up as a seam rather than as an error. Whether that happens is +up to the caller; nothing in the API says so yet. + +### 3. `calculateWithNeighbours` darkens the eight chunks it borrows + +It writes **all nine** chunks back at the end. The eight ring chunks only exchanged light inside the +3×3, so the light they legitimately receive from chunks outside the 3×3 is missing from their result. +Their previously correct light is overwritten with a darker one. + +The middle chunk is not affected, and provably so: a source in chunk (2,0) is at least 17 blocks from +the middle chunk, and no path can be shorter than the direct distance, so level 15 does not survive +the trip. Writing back only the middle chunk would therefore be **cheaper than the current behaviour +and correct at the same time**. The argument is a derivation from the per-block decay, not a +measurement — the byte-identity tests cover a single chunk, not the ring around it. + +This is the concrete form of what was filed under smaller items as "border exchange settles one ring +deep": not an imprecision, a defect with a known fix. + +### 4. Two things that are argued rather than tested + +- **Stale header entries.** `locations` and `timestamps` are `AtomicIntegerArray` now, so a reader + cannot see a stale `0` and turn a present chunk into a regenerated one. That is ruled out by + construction, not by a test — a JMM staleness window cannot be provoked deterministically, because + any harness that tries introduces synchronisation edges of its own. +- **The open-handle limit is no longer a hard cap on descriptors.** It bounds the *cached* files; + a handle in use by a thread stays open beyond it for the duration of that access. Deliberate, and + documented at the field, but it means the limit is a cache size and not a resource guarantee. + +### 5. Smaller items + +- Border exchange between chunks settles one ring deep; a fully converged result over a large area + needs the exchange repeated. Item 3 is the part of this that is outright wrong today. +- Sky light updates re-seed open columns rather than tracking a heightmap incrementally. Measured at + 79.3 → 55.1 µs in the rebuild, with byte identity verified over 240 worlds. +- `SectionOpacity` still builds its table unconditionally for non-uniform sections. Since `69381af` + that costs 8.07 µs instead of 31.33, and the empty section with one source is no longer the row + that loses — but the table is still built whether or not it is read more than once. +- `seed` walks all 4096 positions a second time only to find the emitters. Writing them during the + table build saves about 3.6 µs and one `byte[4096]`, at the price of changing the API of + `SectionOpacity` and both propagators. Left open on purpose for that reason. +- Column opacity is a `List.get(y >> 4)` plus a virtual call rather than one flat `byte[]` (−26 % on + a whole column in the rebuild), and the search neither skips the direction an entry arrived from + (−7 to −16 %) nor tests the level before the opacity (−6 %). + +--- + +## Investigated and deliberately not built + +### Replacing parts of Minestom + +Three "replace this part of Minestom" questions were researched before any code was written. The +answers differed sharply and none was obvious in advance — see [`docs/research/`](docs/research/). + +| Subject | Verdict | +| --- | --- | +| **Palette** | Impossible. `sealed interface Palette permits PaletteImpl` is a hard compiler error, and `Section` is a record holding that exact type. | +| **`InstanceContainer`** | Possible but pointless as asked. It compiles and runs, but four `instanceof InstanceContainer` sites silently take another path for a foreign type, and the tick parallelism the request targeted lives in the global `ThreadDispatcher`, not in the container. | +| **Light engine** | Possible and worth it — this is what was built. | + +The recurring lesson: **sealed-ness decides whether it is possible, and the profile decides whether +it is worth it.** Both have to be checked before designing anything, and neither can be guessed. + +### Importing a foreign light algorithm + +A later round asked whether an algorithm from another engine would close the gap to Minestom that +existed at the time. None would have: the gap was not in the search but around it, and the stage +benchmark has since confirmed that — `propagate` was already faster than Minestom's search, and +closing the gap took a table without allocations, not another algorithm. *Claims about other light +engines* refutes the individual leads. The verdicts below exist so that nobody walks the same road +again. + +| Subject | Verdict | +| --- | --- | +| **Starlight, wholesale** | Nothing left to take. The push-instead-of-pull BFS is already here, and the remainder of Starlight's gain is specific to how vanilla stores light. | +| **Bit-slicing light levels across voxels** | No precedent in any engine. It would be an original design with an unproven benefit. | +| **Parallelising the BFS of a single chunk** | Not worth it. The work is 50–150 µs; handing it to another thread costs more than it saves. Minestom parallelises across chunks, which is the right granularity. | +| **Vector API** | Ruled out by packaging rather than by performance — it would force a JVM flag on every consumer. | +| **A bucket queue (Dial)** | No longer undecided for lack of a measurement. It loses 5–7 % at equal source brightness and wins 32–36 % at mixed brightness, and since `0e8fbb5` the benchmark produces the mixed case: mixed sources cost Aves about 33 % and shrink the lead in that row from 1.30× to 1.06×. The prerequisite is met; the change itself is still not made. | + +--- + +## Documents + +| File | Contents | +| --- | --- | +| [`docs/anvil-chunk-loader.md`](docs/anvil-chunk-loader.md) | Usage, architecture, 20-row comparison with the built-in loader, limits | +| [`docs/light-engine.md`](docs/light-engine.md) | Usage, design, where resources are saved, limits | +| [`docs/benchmarks.md`](docs/benchmarks.md) | How to run the benchmarks and what each measures | +| [`docs/research/`](docs/research/) | The three investigations, with both positions where agents disagreed | diff --git a/build.gradle.kts b/build.gradle.kts index 37a66880..cc262edc 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -2,6 +2,7 @@ plugins { `java-library` `maven-publish` jacoco + alias(libs.plugins.jmh) } group = "net.theevilreaper" @@ -25,14 +26,52 @@ dependencies { implementation(libs.slf4j.api) compileOnly(libs.adventure) + compileOnly(libs.adventure.nbt) + compileOnly(libs.annotations) compileOnly(libs.minestom) testImplementation(libs.adventure) + testImplementation(libs.adventure.nbt) + testImplementation(libs.annotations) testImplementation(libs.minestom) testImplementation(libs.cyano) testImplementation(libs.junit.jupiter) testImplementation(libs.junit.platform.launcher) testRuntimeOnly(libs.junit.jupiter.engine) + + // The benchmarks live in their own source set (src/jmh/java). Nothing declared here reaches the + // main or the test classpath, so the published library never carries a jmh dependency. + jmhImplementation(platform(libs.mycelium.bom)) + jmhImplementation(platform(libs.adventure.bom)) + jmhImplementation(libs.adventure.nbt) + jmhImplementation(libs.annotations) + jmhImplementation(libs.jmh.core) + // Minestom is needed only by the comparison benchmark, which measures our light engine against + // the one the server ships with. It stays out of every other benchmark on purpose, so those + // measure this library and not a registry lookup. + jmhImplementation(libs.minestom) + jmhImplementation(libs.fastutil) + // No jmh annotation processor is declared on purpose. The plugin already generates the harness + // classes with its bytecode generator, and declaring the processor as well makes both of them + // emit the same classes, which leaves the jar with two copies of every benchmark. +} + +jmh { + jmhVersion.set(libs.versions.jmh) + // The tests of this project need a Minestom server, which a benchmark jar cannot start. + includeTests.set(false) + resultFormat.set("JSON") + resultsFile.set(layout.buildDirectory.file("reports/jmh/results.json")) + humanOutputFile.set(layout.buildDirectory.file("reports/jmh/human.txt")) + + // A full run takes the better part of an hour, so a single benchmark has to be reachable + // without editing this file. + // Usage: ./gradlew jmh -Pjmh.include='BitPackerBenchmark.pack' + val include = providers.gradleProperty("jmh.include").orNull + + if (include != null) { + includes.set(listOf(include)) + } } tasks { @@ -41,6 +80,18 @@ tasks { options.release.set(25) } + compileJmhJava { + options.encoding = "UTF-8" + } + + // The benchmarks are compiled by a normal build but never executed by one. A run takes the + // better part of an hour and its numbers are far too noisy on a shared runner to gate anything + // on, while a benchmark that stopped compiling after a refactoring should fail like any other + // source set. + check { + dependsOn(compileJmhJava) + } + jacocoTestReport { dependsOn(rootProject.tasks.test) reports { @@ -52,6 +103,10 @@ tasks { test { finalizedBy(rootProject.tasks.jacocoTestReport) useJUnitPlatform() + // The chunk loader tests allocate payloads of about one mebibyte to cover the external + // chunk file path. Without an explicit heap the worker can die while other build tasks + // run in parallel, which surfaces as an EOFException instead of a test failure. + maxHeapSize = "1g" jvmArgs("-Dminestom.inside-test=true") testLogging { events("passed", "skipped", "failed") diff --git a/docs/anvil-chunk-loader.md b/docs/anvil-chunk-loader.md new file mode 100644 index 00000000..a2d3cb12 --- /dev/null +++ b/docs/anvil-chunk-loader.md @@ -0,0 +1,927 @@ +# Anvil chunk loader + +`AvesAnvilLoader` is a `net.minestom.server.instance.ChunkLoader` implementation that reads and +writes chunks in the Anvil region file format (`r...mca`). It is a drop-in replacement for +`net.minestom.server.instance.anvil.AnvilLoader` and targets servers that load or save many chunks +concurrently, that need a read failure to stay visible instead of being silently replaced by a +freshly generated chunk, and that need to keep serving worlds containing blocks or biomes the +server does not know. It is not a general world-management layer: it handles the `region/` +directory of a single dimension and nothing else (see +[What this loader does NOT do](#what-this-loader-does-not-do)). All references in this document +point at the sources of this branch and at Minestom `2026.06.20-26.1.2`. + +## Status + +> **Experimental.** Every public type of `net.theevilreaper.aves.instance.anvil` and +> `ChunkLoaderFactory` is annotated `@ApiStatus.Experimental`, as is the four-argument +> `AbstractMapProvider.registerInstance` overload. Signatures, class layout and behaviour may still +> change in a minor release. Do not rely on it in code you cannot adapt. + +The annotation is present on all thirteen public types of the package — +[`AvesAnvilLoader`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java), +[`RegionFile`](../src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java), +[`RegionConstants`](../src/main/java/net/theevilreaper/aves/instance/anvil/RegionConstants.java), +[`ChunkCompression`](../src/main/java/net/theevilreaper/aves/instance/anvil/ChunkCompression.java), +[`BitPacker`](../src/main/java/net/theevilreaper/aves/instance/anvil/BitPacker.java), +[`PaletteData`](../src/main/java/net/theevilreaper/aves/instance/anvil/PaletteData.java), +[`PaletteEntryResolver`](../src/main/java/net/theevilreaper/aves/instance/anvil/PaletteEntryResolver.java), +[`BlockPaletteResolver`](../src/main/java/net/theevilreaper/aves/instance/anvil/BlockPaletteResolver.java), +[`BiomePaletteResolver`](../src/main/java/net/theevilreaper/aves/instance/anvil/BiomePaletteResolver.java), +[`NbtReads`](../src/main/java/net/theevilreaper/aves/instance/anvil/NbtReads.java), +[`SectionCodec`](../src/main/java/net/theevilreaper/aves/instance/anvil/SectionCodec.java), +[`AnvilDiagnostics`](../src/main/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnostics.java), +[`AnvilChunkException`](../src/main/java/net/theevilreaper/aves/instance/anvil/AnvilChunkException.java) — +plus [`ChunkLoaderFactory`](../src/main/java/net/theevilreaper/aves/map/provider/ChunkLoaderFactory.java) +and the four-argument [`AbstractMapProvider#registerInstance`](../src/main/java/net/theevilreaper/aves/map/provider/AbstractMapProvider.java). +(`SectorAllocator` is package-private and carries no annotation.) + +**Opt-in only.** Nothing switches to this loader by itself. `AbstractMapProvider` keeps +`net.minestom.server.instance.anvil.AnvilLoader` as its default and only uses `AvesAnvilLoader` when +a `ChunkLoaderFactory` is passed explicitly +([`AbstractMapProvider.DEFAULT_CHUNK_LOADER_FACTORY`](../src/main/java/net/theevilreaper/aves/map/provider/AbstractMapProvider.java), +[`AbstractMapProvider#registerInstance`](../src/main/java/net/theevilreaper/aves/map/provider/AbstractMapProvider.java)). Existing +providers therefore behave exactly as before. + +## Usage + +### Directly on an instance + +```java +import net.kyori.adventure.key.Key; +import net.minestom.server.MinecraftServer; +import net.minestom.server.instance.InstanceContainer; +import net.minestom.server.world.DimensionType; +import net.theevilreaper.aves.instance.anvil.AvesAnvilLoader; + +import java.nio.file.Path; + +public final class Bootstrap { + + public static InstanceContainer createLobby() { + InstanceContainer instance = MinecraftServer.getInstanceManager() + .createInstanceContainer(DimensionType.OVERWORLD); + + Key dimension = DimensionType.OVERWORLD.key(); + AvesAnvilLoader loader = new AvesAnvilLoader(Path.of("worlds", "lobby"), dimension); + + instance.setChunkLoader(loader); + instance.enableAutoChunkLoad(true); + return instance; + } +} +``` + +`AvesAnvilLoader(Path worldRoot, Key dimension)` takes the **world root**, not the region +directory. It resolves `worldRoot/dimensions///region` and falls back to +`worldRoot/region` when only the pre-26.1 layout exists +([`AvesAnvilLoader#resolveRegionDirectory`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). + +The loader implements `AutoCloseable`. `close()` flushes and closes every open region file and +writes the summary line +([`AvesAnvilLoader#close`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java), +[`AvesAnvilLoader#logSummary`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). +Call it on server shutdown. During operation a region file is closed on its own once the last chunk +this loader read from it has been unloaded, and the number of simultaneously open files is capped by +`DEFAULT_OPEN_REGION_LIMIT` (64, configurable through the three-argument constructor). + +A region file is never closed while a thread is reading from or writing to it. Every access +registers itself on the cached handle first; an unload, an eviction or `close()` only drops the +handle from the cache, and the thread that leaves it last performs the actual close. That is why a +chunk load cannot fail because another thread unloaded a chunk of the same region, and why a save +needs no retry when the open-file limit evicts its file mid-write. The cap therefore bounds the +number of *cached* files exactly; the number of open descriptors can exceed it for the duration of +a single access. + +After `close()` the loader refuses further work with an `IllegalStateException` instead of ignoring +it: `loadChunk`, `saveChunk` and `saveChunks` all throw. Returning `null` from a closed loader would +report the chunk as absent and make the server generate a replacement over the stored data, and +ignoring a save would drop chunk data during the very shutdown it belongs to. A task that is already +past that check and reaches the region cache during the close either finds its handle still valid +and finishes normally, or is refused the same way — it can never publish a handle that nothing +closes again. + +### Through a map provider + +`AbstractMapProvider` keeps the Minestom loader as the default, so existing providers are +unaffected. A provider opts in by passing `ChunkLoaderFactory.anvil()` to the four-argument +`registerInstance` overload: + +```java +import net.minestom.server.instance.InstanceContainer; +import net.minestom.server.world.DimensionType; +import net.theevilreaper.aves.file.FileHandler; +import net.theevilreaper.aves.map.BaseMap; +import net.theevilreaper.aves.map.MapEntry; +import net.theevilreaper.aves.map.provider.AbstractMapProvider; +import net.theevilreaper.aves.map.provider.ChunkLoaderFactory; +import net.theevilreaper.aves.util.functional.PathFilter; + +import java.nio.file.Path; + +public final class LobbyMapProvider extends AbstractMapProvider { + + public LobbyMapProvider(FileHandler fileHandler, PathFilter mapFilter) { + super(fileHandler, mapFilter); + } + + public void register(InstanceContainer instance, MapEntry mapEntry) { + // Uses AvesAnvilLoader instead of the Minestom AnvilLoader. + registerInstance(instance, mapEntry, DimensionType.OVERWORLD, ChunkLoaderFactory.anvil()); + } + + @Override + public void saveMap(Path path, BaseMap baseMap) { + // provider specific + } +} +``` + +The relevant signatures are: + +| Member | Declaration | +| --- | --- | +| [`ChunkLoaderFactory#anvil`](../src/main/java/net/theevilreaper/aves/map/provider/ChunkLoaderFactory.java) | `static ChunkLoaderFactory anvil()` | +| [`ChunkLoaderFactory#create`](../src/main/java/net/theevilreaper/aves/map/provider/ChunkLoaderFactory.java) | `ChunkLoader create(MapEntry mapEntry, Key dimension)` | +| [`AbstractMapProvider#registerInstance`](../src/main/java/net/theevilreaper/aves/map/provider/AbstractMapProvider.java) | `protected void registerInstance(InstanceContainer instance, MapEntry mapEntry, RegistryKey dimensionKey, ChunkLoaderFactory loaderFactory)` | +| [`AvesAnvilLoader(Path, Key)`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java) | `public AvesAnvilLoader(Path worldRoot, Key dimension)` | + +The factory receives `MapEntry.getDirectoryRoot()` as the world root and the dimension key of the +instance ([`ChunkLoaderFactory#anvil`](../src/main/java/net/theevilreaper/aves/map/provider/ChunkLoaderFactory.java), +[`AbstractMapProvider#registerInstance`](../src/main/java/net/theevilreaper/aves/map/provider/AbstractMapProvider.java)). +Any other `ChunkLoader` can be supplied the same way, because `ChunkLoaderFactory` is a functional +interface: + +```java +registerInstance(instance, mapEntry, DimensionType.OVERWORLD, + (entry, dimension) -> new AvesAnvilLoader(entry.getDirectoryRoot(), dimension)); +``` + +## Architecture + +### The three-stage pipeline + +Both loading and saving are split into three stages. The point of the split is that no CPU-bound +work happens while a lock is held. Decompression, NBT parsing, palette conversion and compression +are the expensive parts of chunk IO; if they run inside a per-region lock, adding threads adds +contention and nothing else, because every thread touching the same region file has to wait for +them. + +```mermaid +flowchart LR + subgraph load["loadChunk"] + direction LR + L1["1. IO
region lock free
positional read of raw bytes"] + L2["2. Codec
no lock
inflate, NBT parse, palette decode"] + L3["3. Apply
chunk write lock
copy into sections"] + L1 --> L2 --> L3 + end + subgraph save["saveChunk"] + direction LR + S1["1. Snapshot
chunk read lock
clone sections, collect block entities"] + S2["2. Codec
no lock
palette encode, NBT write, deflate"] + S3["3. IO
region lock
allocate sectors, write header entry"] + S1 --> S2 --> S3 + end +``` + +Plain text form: + +``` +loadChunk: read raw bytes -> inflate + parse + decode -> apply to chunk + (no region lock) (no lock at all) (chunk write lock) + +saveChunk: clone sections -> encode + write NBT + deflate -> write sectors + (chunk read lock) (no lock at all) (region lock) +``` + +Concretely: + +* **Load stage 1** — `RegionFile.readRaw` returns the still-compressed payload and takes no lock; + it uses `FileChannel.read(ByteBuffer, long)`, which does not mutate the channel position and is + therefore safe from several threads + ([`RegionFile#readRaw`](../src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java), + [`RegionFile#readEntry`](../src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java), + [`RegionFile#readFully`](../src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java)). + Taking no lock does not mean reading whatever is on disk: a chunk which is rewritten releases its + sector range, and the allocator may hand that range to the next write of any chunk while the reader + is still inside it. Each of the 1024 entries therefore carries a version counter which a writer + raises on entry to and on exit from its critical section. The reader takes the counter, rejects an + odd one, reads the bytes and takes the counter again; anything but an unchanged even counter makes + it start over, and after four attempts it falls back to the writer lock so a chunk which is + rewritten in a loop cannot starve it + ([`RegionFile#readRaw`](../src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java), + the counters in [`RegionFile.versions`](../src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java), + the four in [`RegionFile.OPTIMISTIC_ATTEMPTS`](../src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java)). + Readers are never serialised against each other and never delay a writer. +* **Load stage 2** — decompression and NBT parsing happen in the caller + ([`AvesAnvilLoader#loadChunk`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). + `decodeSections` then returns a `List` + ([`AvesAnvilLoader#decodeSections`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)), + and `loadChunk` calls it **before** it takes the chunk lock. + Everything costly happens here: resolving every palette entry through the resolvers, deriving the + bits per entry, validating the packed arrays and reading the light arrays. The result is a list of + immutable records carrying the section index, the decoded block and biome `PaletteData` and the + two light arrays + ([`AvesAnvilLoader.DecodedSection`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). +* **Load stage 3** — the chunk write lock is taken, each record is transferred via + `DecodedSection.applyTo(chunk)`, the block entities are placed, and the lock is released + ([`AvesAnvilLoader#loadChunk`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java), + [`AvesAnvilLoader#applyBlockEntities`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). + The guarded region performs no parsing and no palette resolution at all — only writes into + `Section.skyLight()`, `Section.blockLight()`, `Section.blockPalette()` and `Section.biomePalette()` + ([`AvesAnvilLoader.DecodedSection#applyTo`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). + This split is the concrete implementation of the pipeline: the record type exists purely to carry + decoded state across the lock boundary. +* **Save stage 1** — the chunk **read** lock is held only long enough to clone every `Section` and + collect the block entities + ([`AvesAnvilLoader#snapshot`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). +* **Save stage 2** — encoding, NBT serialisation and deflate run on the clones, outside every lock + ([`AvesAnvilLoader#snapshot`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java), + [`AvesAnvilLoader#encodeSection`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java), + [`AvesAnvilLoader#saveChunk`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). +* **Save stage 3** — the region lock covers sector allocation, the payload write, the 8-byte header + entry update and, for an oversized chunk, the rename which puts its external file in place. The + header entry decides which of the two storage locations a reader has to follow, so the entry and + the file have to change together; only the bytes of the external file are written before the lock + is taken, into a staging file next to the region file + ([`RegionFile#writeRaw`](../src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java), + [`RegionFile#placeExternal`](../src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java), + [`RegionFile.STAGING_SUFFIX`](../src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java)). + +`supportsParallelLoading()` and `supportsParallelSaving()` both return `true` +([`AvesAnvilLoader#supportsParallelLoading`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java), +[`AvesAnvilLoader#supportsParallelSaving`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)), +so `InstanceContainer` dispatches loads onto virtual threads +([Minestom `InstanceContainer.java:362-372`](#references)). + +### Classes and their responsibility + +Every class in `net.theevilreaper.aves.instance.anvil` has exactly one job, which is what makes +most of the package testable without a running server. + +| Class | Single responsibility | +| --- | --- | +| `RegionConstants` | Layout constants of the region format and pure offset/index arithmetic. No state. | +| `SectorAllocator` | Tracks used sectors in a `BitSet`, first-fit allocation, reuse of freed ranges, overlap detection. No file access. | +| `RegionFile` | Byte container for one `.mca` file: header tables, sector placement, raw read/write, `.mcc` overflow. Knows nothing about NBT or Minestom. | +| `ChunkCompression` | The compression scheme byte: id mapping, external flag, compress/decompress. | +| `BitPacker` | Packing and unpacking of palette indices into `long[]`, and derivation of bits-per-entry. Pure functions. | +| `PaletteData` | Immutable palette + packed indices pair; construction from disk, construction from raw values, unpacking. | +| `PaletteEntryResolver` | Interface between named format entries and numeric server ids. | +| `BlockPaletteResolver` | Block name/properties ↔ Minestom block state id, with air fallback. | +| `BiomePaletteResolver` | Biome name ↔ registry id, with plains fallback and lazily resolved registry. | +| `NbtReads` | Strict accessors for Adventure NBT: a missing or mistyped key is an error, not a default. | +| `SectionCodec` | Palette container (`palette` + `data`) ↔ `PaletteData`, for blocks and biomes. | +| `AnvilDiagnostics` | Throttling of repeated warnings and the counters reported on close. Thread safe. | +| `AnvilChunkException` | The unchecked failure signalling that an existing chunk could not be read. | +| `AvesAnvilLoader` | Orchestration: region file cache, the three stages, block entities, logging, `saveChunks` scheduling. | + +`NbtReads` exists because `CompoundBinaryTag` getters return defaults for missing or mistyped keys. +For chunk data that default is dangerous: a malformed region file would decode into an empty chunk +which then overwrites the real data on the next save +([`NbtReads`](../src/main/java/net/theevilreaper/aves/instance/anvil/NbtReads.java)). +It also avoids the array-tag iterators of Adventure 5.1.1, which stop one entry early +([`NbtReads#longArray`](../src/main/java/net/theevilreaper/aves/instance/anvil/NbtReads.java), +[`NbtReads#intArray`](../src/main/java/net/theevilreaper/aves/instance/anvil/NbtReads.java)). + +### Lazy registry resolution + +`BiomePaletteResolver` does **not** read the biome registry in its constructor. It stores a +`Supplier>` and resolves it on first use, behind double-checked locking on a +`volatile` field holding a private `Registries` record +([`BiomePaletteResolver.registrySupplier`](../src/main/java/net/theevilreaper/aves/instance/anvil/BiomePaletteResolver.java), +[`BiomePaletteResolver.resolved`](../src/main/java/net/theevilreaper/aves/instance/anvil/BiomePaletteResolver.java), +[`BiomePaletteResolver#registries`](../src/main/java/net/theevilreaper/aves/instance/anvil/BiomePaletteResolver.java)). +The record pairs the registry with the id of the fallback biome, so both are published together by +a single volatile write +([`BiomePaletteResolver.Registries`](../src/main/java/net/theevilreaper/aves/instance/anvil/BiomePaletteResolver.java)). + +The reason is a hard ordering constraint. `MinecraftServer.getBiomeRegistry()` dereferences the +static `serverProcess` field (Minestom `MinecraftServer.java:280`), which stays `null` until +`MinecraftServer.init(..)` assigns it through `updateProcess` (`MinecraftServer.java:85-88`, +`:95-99`). Calling it earlier throws. Resolving the registry in the constructor would therefore make +a loader impossible to construct before `MinecraftServer.init(..)` has run — which is exactly when +worlds and map providers are normally set up. Deferring the lookup to the first decoded biome +palette means the loader can be constructed at any point during startup, and the registry is read +only once a chunk is actually being loaded. + +The same constraint is what makes the Minestom `AnvilLoader` hard to use early and hard to unit +test: it reads the registry in a static initialiser (`instance/anvil/AnvilLoader.java:46-48`), so +merely referencing the class before `init` fails. The two-argument constructor of +`BiomePaletteResolver` exists so a test can inject a registry without a server +([`BiomePaletteResolver(AnvilDiagnostics, Supplier)`](../src/main/java/net/theevilreaper/aves/instance/anvil/BiomePaletteResolver.java)). + +`BlockPaletteResolver` needs no such treatment: `Block.fromKey` and `Block.fromStateId` are static +registry lookups performed per palette entry, not cached at class initialisation +([`BlockPaletteResolver#toId`](../src/main/java/net/theevilreaper/aves/instance/anvil/BlockPaletteResolver.java), +[`BlockPaletteResolver#toEntry`](../src/main/java/net/theevilreaper/aves/instance/anvil/BlockPaletteResolver.java)). + +## Comparison with the built-in AnvilLoader + +**How the two sides are referenced.** Minestom references are a path relative to +`net/minestom/server/` plus a line number, at version `2026.06.20-26.1.2`. Aves references name the +member instead — `RegionFile#writeRaw` for a method, `RegionFile.OPTIMISTIC_ATTEMPTS` for a field or +constant, a bare type name for the type itself — and every Aves type lives under +`src/main/java/net/theevilreaper/aves/instance/anvil/` unless the reference says otherwise. + +The asymmetry is deliberate, not an oversight. The Minestom version is pinned, so those files will +never move again and the line number is the most precise pointer available. The Aves sources are the +sources of this branch and change under active work: the concurrency fixes that introduced the +per-entry seqlock and the region handle use count grew `AvesAnvilLoader` by roughly four hundred +lines in a single week, which silently pointed every line number in this document at a blank line or +a stray `*/`. A member name survives everything short of a rename, and a rename at least breaks +loudly. + +| Aspect | Minestom `AnvilLoader` | Aves `AvesAnvilLoader` | Impact | +| --- | --- | --- | --- | +| **Chunk length field** | Writes `4 + 1 + N`: `CHUNK_HEADER_LENGTH = 4 + 1` (`instance/anvil/RegionFile.java:31`), `chunkLength = CHUNK_HEADER_LENGTH + dataBytes.length` (`:99`), `file.writeInt(chunkLength)` (`:123`). | Writes `1 + N`: `int length = COMPRESSION_FIELD_SIZE + stored.length`, then `buffer.putInt(length)` (`RegionFile#writeRaw`, `RegionConstants.COMPRESSION_FIELD_SIZE`). | The format defines the field as compression byte + payload. Minestom's own reader compensates by reading `length - 1` bytes (`RegionFile.java:84`), so its files are self-consistent, but every chunk it writes declares four bytes more than it holds. A spec-conforming reader over-reads up to four bytes of sector padding, and when the payload ends within four bytes of a sector boundary it reads past the allocation. Aves writes the value the format specifies. | +| **Short reads** | `file.read(data)` — the return value is discarded (`instance/anvil/RegionFile.java:85`). `RandomAccessFile.read` may return fewer bytes than requested. | `readFully` loops until the buffer is full and reports EOF as an `IOException` (`RegionFile#readFully`), used for both the header (`RegionFile#readHeader`) and the payload (`RegionFile#readEntry`). | A short read in Minestom leaves the tail of `data` zero-filled and is then handed to the NBT parser, producing a parse error or a truncated chunk with no indication of the cause. Aves either has the full payload or fails with the byte counts in the message. | +| **Status key casing** | Reads `"status"` (`instance/anvil/AnvilLoader.java:133`) and writes `"status"` (`:396`). The vanilla key is `Status`. | Reads `Status` first and falls back to `status` (`AvesAnvilLoader.STATUS_KEY`, `AvesAnvilLoader.LEGACY_STATUS_KEY`, `AvesAnvilLoader#isFullyGenerated`); writes `Status` (`AvesAnvilLoader#snapshot`). | For a vanilla world Minestom's `getString("status")` returns the empty default, which the `status.isEmpty()` branch (`AnvilLoader.java:135`) treats as fully generated — so partially generated vanilla chunks are loaded as if complete, and the warning at `:142` never fires for them. Its own output carries a key vanilla ignores. Aves reads both spellings and writes the vanilla one. | +| **Read failure handling** | `catch (Exception e) { handleException(e); return null; }` (`instance/anvil/AnvilLoader.java:117-120`). | Logs with context, reports to the exception manager and rethrows as `AnvilChunkException` (`AvesAnvilLoader#failedLoad`). | `null` means "chunk absent" to `InstanceContainer`, which then generates a replacement (`instance/InstanceContainer.java:336-343`) that overwrites the unreadable-but-intact data on the next save. Throwing makes `InstanceContainer` complete the load future exceptionally instead (`:367-372`), so the stored bytes are left untouched. | +| **Unknown block** | `Objects.requireNonNull(Block.fromKey(blockName), "Unknown block " + blockName)` (`instance/anvil/AnvilLoader.java:263`). | `BlockPaletteResolver.toId` substitutes `Block.AIR.stateId()` and reports the name once (`BlockPaletteResolver#toId`). | In Minestom one modded or newer-version block name throws an NPE out of `loadSections`, which is swallowed at `:117-120`; the whole chunk is then regenerated and lost on the next save. Aves loses one block state and keeps the chunk. | +| **Unknown biome** | Falls back to `PLAINS_ID` with no report at all (`instance/anvil/AnvilLoader.java:294-296`). | Falls back to plains and reports the name once through the diagnostics (`BiomePaletteResolver#toId`). | Same resulting data, but in Minestom a world referencing biomes the registry does not have is rewritten to plains silently. Aves leaves a log entry and a counter. | +| **Lock granularity while loading** | One `ReentrantLock` per region file (`instance/anvil/RegionFile.java:42`); `readChunkData` holds it across `seek`, the length/compression read, the payload read **and** the decompression plus NBT parse (`:67-92`, parse at `:88`). | `readRaw` holds no lock and uses positional channel reads (`RegionFile#readRaw`); inflate and NBT parse run in the caller (`AvesAnvilLoader#loadChunk`), palette decoding before the chunk lock (`AvesAnvilLoader#decodeSections`). | Minestom serialises the expensive part of every load of the same region behind one lock, so `supportsParallelLoading() == true` yields little for chunks in one region file. In Aves only the byte read touches the file, and it needs no mutual exclusion. | +| **Lock granularity while saving** | The chunk **write** lock is held across the entire serialisation loop for all sections: palettes, block entities, biome lookups, packing (`instance/anvil/AnvilLoader.java:420-519`). Compression itself is outside the region lock (`instance/anvil/RegionFile.java:96-98` before `:104`). | The chunk **read** lock is held only to clone the sections and collect block entities (`AvesAnvilLoader#snapshot`); everything after that works on the clones (`AvesAnvilLoader#encodeSection`, `AvesAnvilLoader#saveChunk`). | Taking the write lock blocks readers as well as writers, for the full duration of encoding a chunk. A read lock over an array of `Section.clone()` calls keeps the chunk readable while it is being serialised. | +| **`saveChunks` default** | `AnvilLoader` does not override it, so `ChunkLoader.saveChunks` applies: one virtual thread per chunk, coordinated by a `Phaser` (`instance/ChunkLoader.java:62-82`). The `catch` branch (`:71-73`) skips `phaser.arriveAndDeregister()`. | Overridden: chunks are grouped by region index, one task per region, concurrency bounded by a `Semaphore` sized to the CPU count (`AvesAnvilLoader#saveChunks`, `AvesAnvilLoader.saveLimit`), results collected in `awaitAll` (`AvesAnvilLoader#awaitAll`). | With a `Throwable` escaping `saveChunk` the registered party is never deregistered, so `phaser.arriveAndAwaitAdvance()` at `ChunkLoader.java:76` never advances and the saving thread blocks for good. Independently, one thread per chunk means every chunk of a region contends for that region's lock while all snapshots are alive at once. Grouping by region removes the contention and the semaphore bounds peak memory. | +| **Chunks over 255 sectors** | `Check.stateCondition(sectorCount >= SECTOR_1MB, "Chunk data is too large to fit in a region file")` (`instance/anvil/RegionFile.java:102`, `SECTOR_1MB = 256` at `:29`), which throws `IllegalStateException` (`utils/validate/Check.java:58-62`). | Payload is written to `c...mcc` next to the region file, the location entry stores an empty payload and the compression byte carries `EXTERNAL_FLAG = 0x80` (`RegionFile#writeRaw`, `RegionFile#externalPath`, `ChunkCompression.EXTERNAL_FLAG`). Reading follows the flag (`RegionFile#readEntry`). | A chunk larger than ~1 MiB compressed cannot be saved at all by Minestom; the exception propagates out of `saveChunk`'s `IOException`-only catch (`AnvilLoader.java:402`). Aves uses the external-file mechanism the format defines and deletes a stale `.mcc` when a chunk shrinks again, inside the same critical section which rewrites the header entry (`RegionFile#writeRaw`, `RegionFile#removeExternal`). | +| **Block entities in single-value sections** | Block entities are collected only inside the `getAll` callback of the non-uniform branch (`instance/anvil/AnvilLoader.java:456-468`); when `section.blockPalette().singleValue() != -1` that branch is skipped entirely (`:436-441`). | `collectBlockEntities` walks every block position of the chunk independently of the palette shape (`AvesAnvilLoader#collectBlockEntities`). | A section whose blocks all share one state id but where some carry NBT or a handler — for example a section of air with handler-marked positions — loses all of its block entities on save in Minestom. | +| **Palette bits-per-entry on load** | `Palette.load(palette, values)` derives bits-per-entry from `palette.length` alone and ignores `values.length` (`instance/palette/PaletteImpl.java:127-132`), called at `instance/anvil/AnvilLoader.java:234` and `:248`. | Derived from the palette size, then verified against the actual `long[]` length (`PaletteData#read`, `BitPacker#resolveBitsPerEntry`); if the two disagree the data is unpacked with the resolved width and written entry by entry (`AvesAnvilLoader#apply`). | The format permits a writer to use a wider bits-per-entry than the palette size requires. Minestom decodes such a section with the wrong stride, producing wrong blocks with no error. Aves detects the mismatch from the array length and decodes with the width that actually fits. | +| **Palette deduplication on save** | Linear search per block: `blockPaletteIndices.indexOf(value)` on an `IntArrayList` (`instance/anvil/AnvilLoader.java:447`); same for biomes with `biomePalette.indexOf(biomeName)` on an `ArrayList` (`:484`). | `PaletteData.encode` assigns indices via `HashMap.computeIfAbsent` (`PaletteData#encode`). | Minestom's per-section cost is O(n·m) for n = 4096 blocks and m = distinct states in the section (biomes: O(64·m) with a deep `BinaryTag` equality per probe). Aves is O(n) hash lookups. This is a structural difference in the algorithm, not a measured figure. | +| **Registry access at class initialisation** | Static fields read the biome registry and the block state count during class init: `BIOME_REGISTRY = MinecraftServer.getBiomeRegistry()`, `PLAINS_ID`, `new CompoundBinaryTag[Block.statesCount()]` (`instance/anvil/AnvilLoader.java:46-48`). | The biome registry is resolved lazily on first use behind a `volatile` field, and the supplier is injectable (`BiomePaletteResolver.resolved`, `BiomePaletteResolver(AnvilDiagnostics, Supplier)`, `BiomePaletteResolver#registries`). Block lookups go through `Block.fromKey` per palette entry (`BlockPaletteResolver#toId`). | Merely referencing `AnvilLoader` before the server registries exist fails in the static initialiser, so the class cannot be constructed during early startup and unit tests must boot a server. In Aves the loader can be constructed before the registries are populated, and the resolver can be tested with a supplied registry. | +| **Logging and diagnostics** | Unthrottled per-chunk `WARN` for partially generated chunks (`instance/anvil/AnvilLoader.java:142`), per-tag `WARN` for invalid sections (`:203`), block entity tags (`:304`) and non-string block properties (`:273-276`). No counters, no summary. | `AnvilDiagnostics` admits only the first occurrence of a distinct name and caps the tracking sets at `MAX_TRACKED_NAMES = 64` (`AnvilDiagnostics.MAX_TRACKED_NAMES`, `AnvilDiagnostics#track`); a partial chunk reports once per loader lifetime (`AnvilDiagnostics#reportPartialChunk`) and a section outside the world is logged at `TRACE` (`AvesAnvilLoader#decodeSections`). A summary line is written on close (`AvesAnvilLoader#logSummary`). | A world with many partial chunks or one unknown modded block produces one log line per chunk in Minestom, which buries everything else. In Aves the same condition produces one line plus a counter, and the cap keeps a corrupt world from growing the tracking sets without bound. | +| **Header write per chunk** | `writeHeader` rewrites the whole 8192-byte header on every dirty save (`instance/anvil/RegionFile.java:182-201`, called at `:131`). | `writeEntry` writes only the 4-byte location and the 4-byte timestamp of the affected index (`RegionFile#writeEntry`). | Minestom rewrites 1024 location and 1024 timestamp entries to change one of each. Beyond the write volume, a crash during that rewrite can damage entries of unrelated chunks; an 8-byte update cannot. | +| **Region header validation** | `readHeader` marks every non-zero location in the bitset, checking only that it stays inside the current sector count (`instance/anvil/RegionFile.java:167-172`, `:234-239`). Overlapping entries are accepted. | Entries pointing into the header or with a zero sector count are dropped (`RegionFile#readHeader`) and `SectorAllocator.reserve` rejects an overlapping range with the conflicting sector in the message (`SectorAllocator#reserve`). | Two location entries claiming the same sectors stay undetected in Minestom until one chunk overwrites the other. Aves fails to open such a file with a message naming the sector. | +| **NBT strictness** | Uses the defaulting getters throughout: `sectionData.getCompound("block_states")` returns an empty compound when absent (`instance/anvil/AnvilLoader.java:239`), and an empty palette list then leaves the section untouched (`:242-249`). | `NbtReads` reports a missing or mistyped key as an `IOException` naming the key, the expected type and the actual type (`NbtReads#longArray`, `NbtReads#missing`); `SectionCodec` rejects empty palettes (`SectionCodec#decode`, `SectionCodec#decodeBiomes`). | In Minestom a truncated or malformed section silently loads as untouched (air) and is written back that way. In Aves the same input fails the load, so the stored bytes survive. | +| **Region file lifecycle** | Opened inside `alreadyLoaded.computeIfAbsent(...)`, i.e. blocking file IO inside a `ConcurrentHashMap` mapping function (`instance/anvil/AnvilLoader.java:179-194`); closed when the last chunk of the region unloads (`:557-584`). | Opened outside the mapping function, published with `putIfAbsent`, and a losing race closes the redundant handle (`AvesAnvilLoader#acquireRegion`); a file is closed once the last chunk this loader loaded is unloaded, with a hard cap on open files as a backstop (`AvesAnvilLoader.DEFAULT_OPEN_REGION_LIMIT`); a handle in use is only dropped from the cache and closed by its last user. | `computeIfAbsent` holds the bin lock for the duration of the mapping function; performing file IO there blocks other keys hashing to the same bin. Also, `unloadChunk` is called for chunks the loader never loaded (documented at `instance/ChunkLoader.java:102-108`), which makes a plain reference count unreliable — Aves therefore tracks only the chunks it loaded itself and additionally caps the number of open files. | +| **Instance-level and unknown chunk tags** | `loadInstance`/`saveInstance` read and write `level.dat` (`instance/anvil/AnvilLoader.java:96-107`, `:332-343`). Chunk tags other than `Heightmaps`, `sections` and `block_entities` are kept in the chunk tag handler (`:144-151`) and written back on save (`:390`); heightmaps are restored (`:140`). | Neither method is overridden. `snapshot` builds a fixed set of keys: `DataVersion`, `xPos`, `zPos`, `yPos`, `Status`, `LastUpdate`, `sections`, `block_entities` (`AvesAnvilLoader#snapshot`). | This one favours Minestom. Saving a vanilla chunk with the Aves loader drops `Heightmaps`, `structures`, `block_ticks`, `fluid_ticks`, `PostProcessing` and any other chunk-level tag, and `level.dat` is not touched at all. See the next section. | + +Twenty rows. Every reference above was read in the sources of the stated versions. + +## Which of those rows carry the argument + +The table is flat by construction — every row gets one line, whether it changes what a server does +or tidies a log message. Sorted by consequence they fall into four groups. + +**Silent data loss.** The rows that change what ends up on disk: block entities dropped from uniform +sections, the palette stride derived from the palette length alone, the discarded return value of +`read`, the length field written four bytes too large. None of these announce themselves — the world +loads, the chunk looks fine, and the damage surfaces later or in another tool. These are the reason +the loader exists. + +**Failure that destroys the original.** A read error returning `null` means "chunk absent" to +`InstanceContainer`, which generates a replacement and overwrites the intact-but-unreadable bytes on +the next save. This one row turns a recoverable problem into an unrecoverable one. + +**Concurrency.** Both loaders report `supportsParallelLoading() == true`. Only one of them means it. +This is the group with the largest measured effect, and the diagrams below are about it. + +**Everything else** — logging volume, registry access at class-initialisation time, header write +volume — is real but bounded. A server survives all of it. + +### Why parallel loading is nominal in Minestom + +Both loaders do the same work per chunk: read bytes, inflate, parse NBT, decode palettes. The +difference is which of those steps happens while the region file's lock is held. + +```mermaid +flowchart LR + subgraph mine["Minestom · RegionFile.readChunkData"] + direction TB + M1["seek + read length"] + M2["read payload"] + M3["inflate"] + M4["parse NBT"] + M1 --> M2 --> M3 --> M4 + end + subgraph aves["Aves · RegionFile.readRaw + caller"] + direction TB + A1["positional read"] + A2["inflate"] + A3["parse NBT"] + A4["decode palettes"] + A1 --> A2 --> A3 --> A4 + end +``` + +In Minestom all four steps sit inside one `ReentrantLock` held per region file +(`instance/anvil/RegionFile.java:42`, parse at `:88`). In Aves only the first one touches the file, +and it needs no mutual exclusion at all: `FileChannel.read(ByteBuffer, position)` does not move the +channel position, so two readers of different chunks do not interfere. Inflate, parse and palette +decode run in the caller. + +The consequence appears as soon as two threads want chunks from the same region — which is exactly +what loading a spawn area does, since a region file holds 32×32 chunks: + +```mermaid +sequenceDiagram + participant T1 as Thread 1 + participant T2 as Thread 2 + participant R as Region file + Note over T1,R: Minestom — the lock spans the expensive part + T1->>R: acquire lock + T1->>R: read bytes, inflate, parse NBT + T2->>R: acquire lock — blocked for all of it + R-->>T1: release + R-->>T2: granted + T2->>R: read bytes, inflate, parse NBT +``` + +```mermaid +sequenceDiagram + participant T1 as Thread 1 + participant T2 as Thread 2 + participant R as Region file + Note over T1,R: Aves — only the byte read is ordered + T1->>R: positional read + T2->>R: positional read, concurrent + R-->>T1: bytes + R-->>T2: bytes + T1->>T1: inflate, parse, decode + T2->>T2: inflate, parse, decode +``` + +Since inflate and NBT parsing dominate the load path, putting them inside the lock means extra +threads mostly queue. That is what "nominal parallelism" means here, and the next section measures it. + +### Why the save path blocks readers in Minestom + +The same question on the write side, with a different answer. Minestom holds the chunk's **write** +lock across the serialisation of every section — palettes, block entities, biome lookups, packing +(`instance/anvil/AnvilLoader.java:420-519`). A write lock excludes readers as well as writers, so +for the whole duration of encoding, nothing else may look at that chunk. + +Aves takes the **read** lock, clones the sections, and releases it. Everything after that works on +copies: + +```mermaid +flowchart TB + subgraph mineS["Minestom · saveChunk"] + direction TB + MW["chunk WRITE lock"] + MW --> MS["encode all sections
palettes, block entities, biomes, packing"] + MS --> MR["release"] + MB["other readers of this chunk: blocked throughout"] + MS -.-> MB + end + subgraph avesS["Aves · saveChunk"] + direction TB + AR["chunk READ lock"] + AR --> AC["clone sections"] + AC --> AU["release"] + AU --> AE["encode + deflate on the clones
no chunk lock held"] + AB["other readers of this chunk: admitted"] + AC -.-> AB + end +``` + +### Why one exception in `saveChunks` hangs the saving thread + +`AnvilLoader` does not override `saveChunks`, so the interface default applies: one virtual thread +per chunk, coordinated by a `Phaser` (`instance/ChunkLoader.java:62-82`). Its `catch` branch +(`:71-73`) returns without calling `phaser.arriveAndDeregister()`. + +```mermaid +flowchart TB + subgraph mineB["Minestom · ChunkLoader.saveChunks default"] + direction TB + P["phaser.register() per chunk"] + P --> TH["one virtual thread per chunk — unbounded"] + TH --> OK["success: arriveAndDeregister"] + TH --> ERR["exception: caught, NOT deregistered"] + OK --> W["arriveAndAwaitAdvance"] + ERR --> HANG["party never arrives
saving thread blocks for good"] + end + subgraph avesB["Aves · saveChunks"] + direction TB + G["group chunks by region index"] + G --> S["one task per region,
bounded by a Semaphore"] + S --> C["collect every result in awaitAll"] + C --> F["a failure surfaces as a failed future"] + end +``` + +Two independent problems in one row. The `Phaser` branch is a liveness bug: a single `Throwable` +escaping `saveChunk` blocks the saving thread permanently. The unbounded thread-per-chunk is a +memory one: every chunk of a region contends for that region's lock while all snapshots are alive at +once. Grouping by region removes the contention, and the semaphore bounds the peak. + +## Performance and memory + +Two kinds of statement appear below. The **structural** differences are visible in the source of both +implementations and are marked as such. The **measured** ones come from the JMH benchmarks in +`src/jmh` (see [`benchmarks.md`](benchmarks.md)) and name their setup and their spread. Where a cost +is called dominant without a benchmark name attached, it comes from a one-off micro-measurement taken +while designing the loader — treat those as orders of magnitude. + +Two facts shaped every decision below. In the load path, zlib inflate plus NBT parsing dominate — +palette handling is a small fraction of the total. In the save path, deflate dominates everything +else. Optimising the palette would therefore have been pointless; keeping compression and parsing +**out of the locks** is where the time actually is. + +### Measured: concurrent readers of one region file + +`RegionFileComparisonBenchmark` measures the region file of Aves against the one Minestom ships with, +on the same stored bytes, through the same Adventure writer at the same compression level, from a +stored chunk to a parsed compound. It lives in `net.minestom.server.instance.anvil` because +Minestom's `RegionFile` is package-private — the same reason the light comparison lives in Minestom's +light package. Minestom's `AnvilLoader` itself cannot be reached from a benchmark fork at all: its +static fields read the biome registry and the block state count, so the class initialiser fails +before any measurement starts. The region file reads no registry and is measurable directly. + +``` +java -jar build/libs/aves-*-jmh.jar "RegionFileComparisonBenchmark.(aves|minestom)Read" \ + -f 1 -wi 3 -i 5 -t -p distinctStates=200 +``` + +| Threads | Aves | Minestom | +| ---: | ---: | ---: | +| 1 | 1 089 ± 48 µs/op | 1 045 ± 112 µs/op | +| 2 | 1 174 ± 71 µs/op | 103 437 ± 856 306 µs/op | +| 4 | 1 370 ± 200 µs/op | 302 704 ± 674 429 µs/op | +| 8 | 2 282 ± 248 µs/op | 297 075 ± 593 563 µs/op | + +Repeated at four threads with two forks and ten iterations, which tightens the spread considerably: +Aves **1 325.6 ± 21.1 µs/op** against Minestom **359 690.8 ± 97 498.3 µs/op**, a factor of **271**. + +Three things this measurement says, including the ones that do not flatter Aves: + +- **On one thread there is no advantage.** Minestom is marginally ahead, well inside the spread. The + design pays off under contention and nowhere else. +- **Aves degrades gently.** From one to eight threads its cost grows by a factor of 2.1, which is + what sharing a disk looks like. Minestom does not degrade, it collapses. +- **The size of the collapse is not fully explained.** Pure serialisation of a 1 045 µs operation + across four threads predicts roughly 4 200 µs, not 360 000. The measured effect exceeds that by + almost two orders of magnitude, which points at lock convoying or scheduler interaction rather than + plain queueing. The direction is reproduced across two independent runs and the magnitude is + stable; the mechanism behind the magnitude has not been investigated. Do not quote the factor as if + it were understood. + +The same measurement as a picture. On one thread the two loaders sit on top of each other; from two +threads onwards one of them stays roughly where it was and the other leaves the chart. + +```mermaid +%%{init: {"themeVariables": {"xyChart": {"plotColorPalette": "#56B4E9, #E69F00"}}}}%% +xychart-beta + title "Reading one region file: Aves (flat, along the bottom) against Minestom (climbing)" + x-axis "Threads reading from the same region file" [1, 2, 4, 8] + y-axis "Microseconds per read, lower is better" 0 --> 320000 + line [1089, 1174, 1370, 2282] + line [1045, 103437, 302704, 297075] +``` + +`xychart-beta` cannot draw a legend, so it has to be said instead: the line running along the bottom +is **Aves** (blue in every chart of this document), the one climbing away from it is **Minestom** +(orange). Aves is not actually flat there. It only looks flat because the scale has to reach 300 000 +to fit the other line at all. Its own shape is the next chart, and it is the same four numbers. + +```mermaid +%%{init: {"themeVariables": {"xyChart": {"plotColorPalette": "#56B4E9, #E69F00"}}}}%% +xychart-beta + title "The same four Aves numbers, on a scale that fits them" + x-axis "Threads reading from the same region file" [1, 2, 4, 8] + y-axis "Microseconds per read, lower is better" 0 --> 2500 + line [1089, 1174, 1370, 2282] +``` + +**Why the first chart looks like that.** Picture a shop with a single till. Minestom's region file +has one lock per file, and it holds that lock not only while it fetches the bytes from disk, but also +while it unpacks them and reads the structure inside them — and the unpacking and the reading are +nearly all of the work. So one customer occupies the till for the entire purchase, and everybody else +stands in the queue. Adding threads adds people to the queue; it does not add tills. Aves holds the +lock only for fetching the bytes and does the unpacking and reading outside it, so several threads +are served at the same time. That is also why the second chart still rises rather than staying level: +those threads do share one disk, and going from one to eight of them costs a factor of 2.1 — but they +spend that time working, not waiting. + +Two things the charts are not allowed to imply. First, a drawn line is a mean without its spread, and +at two threads the Minestom measurement is 103 437 ± 856 306 µs/op — an uncertainty eight times the +value itself. That point says "sometimes catastrophic", not "this is what it costs"; the four-thread +control run with two forks is the trustworthy one. Second, standing in a queue does not explain how +high the line goes, as the third bullet above says. + +### Measured: saving a chunk + +`ChunkSaveComparisonBenchmark` runs the whole save path of both loaders over the same chunk, varying +how many distinct block states a section holds — the axis on which the palette deduplication differs +(linear scan per block against a hash lookup). Compression is held identical on both sides, so this +measures the loaders and not the zlib level. + +``` +java -jar build/libs/aves-*-jmh.jar "ChunkSaveComparisonBenchmark.(aves|minestom)Save" -f 1 -wi 3 -i 6 +``` + +| Distinct states | Aves | Minestom | +| ---: | ---: | ---: | +| 1 | 968 ± 52 µs/op | 918 ± 39 µs/op | +| 16 | 3 826 ± 279 µs/op | 3 959 ± 227 µs/op | +| 64 | 5 555 ± 305 µs/op | 6 435 ± 421 µs/op | +| 256 | 11 427 ± 980 µs/op | 12 095 ± 1 326 µs/op | +| 1 024 | 41 361 ± 4 082 µs/op | 47 273 ± 3 964 µs/op | + +Here the two lines lie almost on top of each other — and that, rather than a gap, is the finding. + +```mermaid +%%{init: {"themeVariables": {"xyChart": {"plotColorPalette": "#56B4E9, #E69F00"}}}}%% +xychart-beta + title "Saving one chunk: Aves (blue) against Minestom (orange)" + x-axis "Different block states in one section" [1, 16, 64, 256, 1024] + y-axis "Microseconds per save, lower is better" 0 --> 50000 + line [968, 3826, 5555, 11427, 41361] + line [918, 3959, 6435, 12095, 47273] +``` + +No legend again: at the right-hand end the lower of the two lines is Aves (blue) and the upper one is +Minestom (orange). Towards the left they are hard to tell apart, and at the very first point Minestom +is the lower of the two. + +**Why so little happens here.** Saving a chunk is mostly one single job: squeezing the data small +before it is written. Both loaders hand that job to the same library at the same setting, so most of +the time is identical by construction. What differs is how each of them writes down the list of block +types a section contains. Minestom searches the list it has built so far once for every single block; +Aves keeps a lookup table and asks it directly — the difference between paging through a book for a +word and going to its index. That really is cheaper, and the more different block types a section +holds the more often it matters, which is why the two lines part towards the right. But it is a small +part of a large job, and no arrangement of it changes what the squeezing costs. + +**What the chart must not be read as saying.** Its lines are means, and every point also carries a +spread that a line chart cannot draw. At most of the points that spread is wider than the gap you can +see, so the chart shows a difference the measurement does not establish. The paragraph below names +the two points that survive it. + +This is a far smaller effect than the read path, and it deserves to be read conservatively. At one +distinct state Minestom is ahead. At 64 and 1 024 states Aves is ahead by 1.16× and 1.14×, and those +are the only two rows whose spreads do not overlap. The quadratic-versus-linear difference in the +palette is therefore real but modest at realistic section contents — deflate dominates the save path, +and no palette algorithm changes that. + +The compression **level** is a separate matter and is measured separately in the same benchmark +(`compressAvesLevel` against `compressMinestomLevel`): at 256 distinct states the level-2 default of +this loader compresses roughly 2.4× faster than level 6, and 3.1× faster at 1 024. That is a +configuration choice available to either loader, not a property of this one, and it costs roughly +3 % in stored bytes. + +### Time + +| Property | Minestom | Aves | Why it matters | +|---|---|---|---| +| Work inside the region lock (read) | `readChunkData` holds one `ReentrantLock` across seek, read, decompression and NBT parsing (`instance/anvil/RegionFile.java:42`, `:57-89`) | The read takes no lock at all; only a read that keeps racing a writer falls back to it. Decompression, NBT parsing and palette conversion run in the caller (`RegionFile#readRaw`, `AvesAnvilLoader#loadChunk`) | This is the whole reason `supportsParallelLoading()` is worth reporting. With the dominant cost inside the lock, extra threads queue instead of working. | +| Concurrent readers of one region | Serialised by the single lock, plus `RandomAccessFile.seek` makes shared use unsafe | `FileChannel.read(ByteBuffer, position)` does not touch the channel position, so readers of different chunks proceed in parallel (`RegionFile#readFully`) | Loading a spawn area touches many chunks of the same region file at once. | +| Chunk lock held while saving | Write lock over the entire serialisation of all sections (`instance/anvil/AnvilLoader.java:420-519`) | Read lock only while cloning sections into a snapshot; serialisation and compression happen after it is released (`AvesAnvilLoader#snapshot`) | A write lock blocks readers of that chunk; on `saveChunksToStorage` this stalls the tick thread for the duration of the serialisation. | +| Header write per chunk save | Rewrites the full 8192-byte header whenever it is dirty (`RegionFile.java:182-196`) | Patches the 4-byte location entry and the 4-byte timestamp entry only (`RegionFile#writeEntry`) | 8192 bytes versus 8 bytes per save. It also narrows the window in which a crash can damage unrelated entries. | +| Palette deduplication on save | `IntArrayList.indexOf(value)` per block, i.e. a linear scan for each of the 4096 blocks of a section (`instance/anvil/AnvilLoader.java:447`), and the same for biomes (`:484`) | Hash-based index assignment, one lookup per block (`PaletteData#encode`) | Quadratic versus linear in the palette size. Sections with large palettes are the worst case. | +| Re-packing on load | `Palette#load` derives bits per entry from the palette length alone (`instance/palette/PaletteImpl.java:128-129`) | The stored `long[]` is validated and handed over unchanged when its bit width matches; only a mismatching file is unpacked and re-applied (`AvesAnvilLoader#apply`) | The common case avoids an unpack/repack round trip entirely. The uncommon case is decoded correctly instead of silently misread. | + +### Memory + +| Property | Minestom | Aves | Why it matters | +|---|---|---|---| +| Concurrency of `saveChunks` | Interface default starts one virtual thread **per chunk** (`instance/ChunkLoader.java:62-82`) | Chunks are grouped per region, one task per group, bounded by a `Semaphore` sized to the available processors (`AvesAnvilLoader#saveChunks`, `AvesAnvilLoader.saveLimit`) | The number of chunk snapshots and compressed byte arrays alive at once is bounded by the permit count instead of by the number of chunks being saved. | +| Uniform sections | Written as a full palette container | Collapsed to a single palette entry with no data array (`PaletteData#single`) | A section of pure air or pure stone stores one entry instead of a 4096-entry index array. | +| Repeated array reads | — | `NbtReads` copies each array tag once and never calls `value()` twice | `value()` on an array tag copies on every call; a 4096-entry `long[]` is 32 KiB per copy. | +| Open file handles | Closed when the last chunk of a region unloads, using a reference count that the interface documents as unreliable (`instance/ChunkLoader.java:102-108`) | Closed when the last chunk **this loader loaded** is unloaded, plus a hard cap on open files as a backstop (`AvesAnvilLoader.DEFAULT_OPEN_REGION_LIMIT`); never closed under a thread that is still reading or writing, and never opened again after `close()` | Unload calls arrive for foreign chunks, so a count alone either leaks handles or closes files still in use. The cap bounds the worst case regardless. | +| Block state cache | `static CompoundBinaryTag[]` sized by `Block.statesCount()`, populated without synchronisation (`instance/anvil/AnvilLoader.java:48`, `:526-531`) | No global cache; palette entries are built per section (`BlockPaletteResolver#toEntry`) | Trades a small amount of repeated work for no shared mutable state and no class-loading-time allocation proportional to the block registry. | + +### What is not faster + +Being explicit about this, because the table above is one-sided by construction: + +- **On a single thread this loader is not faster at all.** Measured on the region file, Minestom + comes out marginally ahead — 1 045 ± 112 against 1 089 ± 48 µs/op, which is inside the spread of + both. Everything the design buys is bought under contention. A single-threaded workload that reads + one chunk at a time gains nothing here and pays for the version counter, the stricter NBT reads and + the length validation. +- Aves does **not** parse NBT faster — both use adventure-nbt 5.1.1, and parsing is the largest single + cost in the load path. +- Aves does **not** compress faster — both use `java.util.zip`, and deflate dominates the save path. +- Aves writes **more** data per chunk in one respect: block entities are collected for uniform + sections too, which Minestom skips (see the comparison table). That is a correctness fix, not a + saving. +- The palette representation is a value record, so a section snapshot allocates. Minestom mutates a + palette in place. Aves trades that allocation for the ability to build sections without holding + the chunk lock. + +## What this loader does NOT do + +Stated plainly, because each of these is a reason to keep using another loader or another tool. + +* **No `entities/` or `poi/` region handling.** Only the `region/` directory is read and written. + Entity and point-of-interest region files of a vanilla world are ignored and are neither migrated + nor kept in sync. The Minestom `AnvilLoader` does not handle them either. +* **No DataFixer and no DataVersion migration.** `MinecraftServer.DATA_VERSION` is written into + every saved chunk ([`AvesAnvilLoader.dataVersion`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java), + written by [`AvesAnvilLoader#snapshot`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)), + but the stored `DataVersion` of a chunk being read is never inspected. Data from an older world + version is interpreted with the current schema. Convert worlds with the vanilla client or + another tool first. +* **No LZ4 (compression type 4) and no custom compression (type 127).** `ChunkCompression.fromId` + accepts gzip (1), zlib (2) and uncompressed (3), with the external bit `0x80` masked off; every + other id raises `The compression scheme is not supported. Only gzip (1), zlib (2) and none (3) + can be read` ([`ChunkCompression#fromId`](../src/main/java/net/theevilreaper/aves/instance/anvil/ChunkCompression.java)). + Unsupported compression therefore fails with an explicit error instead of being misread as + another scheme. Saving always uses zlib + ([`AvesAnvilLoader#saveChunk`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). +* **No corruption recovery and no header rebuilding.** A header shorter than 8192 bytes, or a + location table with overlapping sector ranges, fails the open + ([`RegionFile#readHeader`](../src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java), + [`SectorAllocator#reserve`](../src/main/java/net/theevilreaper/aves/instance/anvil/SectorAllocator.java)). + There is no scan-and-repair mode, no orphaned-sector reclamation and no defragmentation; freed + sectors are reused but the file is never shrunk + ([`SectorAllocator#free`](../src/main/java/net/theevilreaper/aves/instance/anvil/SectorAllocator.java)). +* **No `level.dat` handling.** `loadInstance` and `saveInstance` are not overridden, so world + metadata (seed, spawn, game rules, world age) is neither read nor written. `LastUpdate` is + written as a constant `0` + ([`AvesAnvilLoader#snapshot`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). +* **No preservation of unknown chunk-level tags.** `snapshot` writes a fixed key set + ([`AvesAnvilLoader#snapshot`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)), + so `Heightmaps`, `structures`, `block_ticks`, `fluid_ticks` and everything else present in a + vanilla chunk are lost when that chunk is saved. Heightmaps are not restored on load either. + Use this loader for worlds the server owns, not as an editor for vanilla worlds you intend to + open in the client again. + +## Error handling and world consistency + +The governing rule: **a chunk that exists on disk but cannot be read must not be reported as +absent.** + +`ChunkLoader.loadChunk` uses `null` for "this loader has no data for that chunk". `InstanceContainer` +reacts by generating a replacement chunk, caching it and firing the load event +(`InstanceContainer.java:336-343`). That replacement is a normal, dirty chunk, so the next +`saveChunk` writes it over the bytes that failed to read. A transient IO error, a temporarily +unavailable mount or a parser bug therefore does not merely fail a load in Minestom — it destroys +the data it failed to read, without an error visible to the operator beyond one handled exception. + +`AvesAnvilLoader.loadChunk` returns `null` only for the two genuinely-absent cases: no region file +([`AvesAnvilLoader#acquireRegion`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java) +returns `null` without `create`) and no location entry for the chunk +([`RegionFile#readEntry`](../src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java) +returns `null`, handled in +[`AvesAnvilLoader#loadChunk`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). +A chunk that is present but not fully generated also returns `null` after a throttled warning +([`AvesAnvilLoader#isFullyGenerated`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)), +which matches the intent of the format. Every other failure — malformed header, short read, +unsupported compression, broken NBT, palette index out of range — is logged with context, handed to +the exception manager and rethrown as `AnvilChunkException` +([`AvesAnvilLoader#failedLoad`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). +`InstanceContainer` then completes the load future exceptionally instead of generating +(`InstanceContainer.java:367-372`), the chunk stays unloaded, and nothing overwrites it. + +`AnvilChunkException` is an unchecked exception so it can cross the `ChunkLoader` interface, which +declares no checked exceptions +([`AnvilChunkException`](../src/main/java/net/theevilreaper/aves/instance/anvil/AnvilChunkException.java)). + +`saveChunk` deliberately does **not** throw. It logs at error level, increments the error counter +and reports to the exception manager +([`AvesAnvilLoader#saveChunk`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). +A failed save has already lost the in-memory state either way; propagating would additionally abort +the surrounding save of every other chunk. The one exception is a save on a **closed** loader: that +is a lifecycle error of the caller rather than a broken chunk, so it propagates as an +`IllegalStateException` and is not counted as a failed chunk. In `saveChunks` a task that fails is reported per group +in `awaitAll` ([`AvesAnvilLoader#awaitAll`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)), +and the error count surfaces again in the summary written by `close()`. + +## Logging + +Only three classes own a logger: + +| Class | Logger | Why | +| --- | --- | --- | +| `AvesAnvilLoader` | yes ([`AvesAnvilLoader.LOGGER`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)) | It is the only layer that knows chunk coordinates, region directory and dimension. | +| `BlockPaletteResolver` | yes ([`BlockPaletteResolver.LOGGER`](../src/main/java/net/theevilreaper/aves/instance/anvil/BlockPaletteResolver.java)) | Reports a substituted block name once; the loader never sees the substitution. | +| `BiomePaletteResolver` | yes ([`BiomePaletteResolver.LOGGER`](../src/main/java/net/theevilreaper/aves/instance/anvil/BiomePaletteResolver.java)) | Same, for biomes. | + +`RegionFile`, `SectorAllocator`, `BitPacker`, `ChunkCompression`, `NbtReads`, `PaletteData`, +`SectionCodec`, `RegionConstants` and `AnvilDiagnostics` deliberately have none. They are leaf +classes that do not know which chunk, region or dimension they are working on, so any line they +logged would be context-free. Instead they throw with the facts they do have — the offending key +and its actual type ([`NbtReads#missing`](../src/main/java/net/theevilreaper/aves/instance/anvil/NbtReads.java)), +the declared length against the sector allocation +([`RegionFile#readEntry`](../src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java)), +the long count against the entry count +([`PaletteData#read`](../src/main/java/net/theevilreaper/aves/instance/anvil/PaletteData.java)), +the conflicting sector ([`SectorAllocator#reserve`](../src/main/java/net/theevilreaper/aves/instance/anvil/SectorAllocator.java)). +The loader catches these and adds the context. + +**Message schema.** Every loader message that concerns a chunk ends with the same trailer, so logs +can be grepped and parsed uniformly: + +``` +... chunk=[{},{}] region={} dim={} +``` + +for example +`Failed to load the chunk chunk=[{},{}] region={} dim={}` +([`AvesAnvilLoader#failedLoad`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). +Messages that concern a whole region omit the `chunk=` part and keep `region={} dim={}` +([`AvesAnvilLoader(Path, Key, int)`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java), +[`AvesAnvilLoader#closeQuietly`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java), +[`AvesAnvilLoader#close`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java), +[`AvesAnvilLoader#acquireRegion`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java), +[`AvesAnvilLoader#awaitAll`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). + +**Throttling.** `AnvilDiagnostics` decides whether a report is emitted. `reportUnknownBlock` and +`reportUnknownBiome` return `true` only for the first occurrence of a distinct name, and only while +fewer than `MAX_TRACKED_NAMES = 64` names are tracked in that category +([`AnvilDiagnostics.MAX_TRACKED_NAMES`](../src/main/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnostics.java), +[`AnvilDiagnostics#track`](../src/main/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnostics.java)). +The cap matters because a broken or heavily modded world can contain an unbounded number of distinct +unknown names, which would otherwise grow the tracking sets indefinitely. `reportPartialChunk` and +`reportSectionOutOfRange` use an `AtomicBoolean` and fire at most once per loader +([`AnvilDiagnostics#reportPartialChunk`](../src/main/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnostics.java), +[`AnvilDiagnostics#reportSectionOutOfRange`](../src/main/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnostics.java)) — +the loader currently calls only the first of the two; a section outside the world is logged at +`TRACE` without going through the diagnostics. +The sets are `ConcurrentHashMap.newKeySet()` and the counters are `LongAdder`, so reporting from +many loader threads is safe +([`AnvilDiagnostics()`](../src/main/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnostics.java)). + +Consequence to be aware of: once 64 distinct unknown block names have been seen, further distinct +names are substituted silently. The counters still rise, and `unknownBlockCount()` saturates at 64. + +**Close summary.** `close()` writes one line with loaded chunks, saved chunks, errors, distinct +unknown blocks and distinct unknown biomes, plus the region/dimension trailer. It is logged at +`WARN` when the error count is greater than zero and at `INFO` otherwise, so a shutdown that lost +chunks does not read like a clean one +([`AvesAnvilLoader#logSummary`](../src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java)). + +Levels in use: `INFO` for open and clean close, `WARN` for throttled data problems and a close with +errors, `ERROR` for a failed chunk load/save and a region file that could not be closed, `DEBUG` +when a region file is opened, `TRACE` for chunk unloads and skipped out-of-world sections. + +## Testing + +The full suite is **575 tests, 0 failures, 0 errors**. Thirteen of those classes cover the anvil +package, contributing **178 executed tests**, plus 3 for the factory. + +The two columns differ because `@ParameterizedTest` methods expand into one executed test per +argument set. "Declared" counts annotated methods in the source; "executed" is what JUnit actually +ran, taken from `build/test-results/test/TEST-*.xml`. + +| Test class | Declared methods | Executed tests | Needs a Minestom server | +| --- | --- | --- | --- | +| `BitPackerTest` | 12 (8 + 4 parameterized) | 34 | no | +| `PaletteDataTest` | 17 (16 + 1 parameterized) | 22 | no | +| `ChunkCompressionTest` | 14 (11 + 3 parameterized) | 21 | no | +| `AvesAnvilLoaderIntegrationTest` | 19 | 19 | **yes** | +| `NbtReadsTest` | 15 | 15 | no | +| `RegionFileTest` | 14 | 14 | no | +| `SectionCodecTest` | 13 | 13 | no | +| `AnvilDiagnosticsTest` | 10 | 10 | no | +| `SectorAllocatorTest` | 9 (8 + 1 parameterized) | 10 | no | +| `RegionFileConcurrencyTest` | 6 | 6 | no | +| `AnvilDiagnosticsConcurrencyTest` | 5 | 5 | no | +| `AvesAnvilLoaderLifecycleTest` | 5 | 5 | **yes** | +| `AvesAnvilLoaderConcurrencyTest` | 4 | 4 | **yes** | +| **Total (anvil package)** | **143** | **178** | | +| `map/provider/ChunkLoaderFactoryTest` | 3 | 3 | no | + +**Layers that need no server.** Ten of the thirteen classes import nothing from `net.minestom`. That +is a direct consequence of the class split: `RegionConstants`, `SectorAllocator`, `RegionFile`, +`ChunkCompression`, `BitPacker`, `PaletteData`, `SectionCodec`, `NbtReads` and `AnvilDiagnostics` +have no dependency on the server or its registries. `SectionCodecTest` exercises the codec through a +stub `PaletteEntryResolver`, so palette encoding and decoding are verified without touching the block +or biome registry. `RegionFileTest` works on real files in a `@TempDir`. `ChunkLoaderFactoryTest` +only checks which loader type a factory produces and which path it receives, so it needs no server +either. + +`RegionFileConcurrencyTest` and `AnvilDiagnosticsConcurrencyTest` hammer the same file and the same +diagnostics from several threads and stay server-free for the same reason. + +**The layer that does.** Three classes need one: `AvesAnvilLoaderIntegrationTest`, +`AvesAnvilLoaderLifecycleTest` and `AvesAnvilLoaderConcurrencyTest`. The first is annotated +`@ExtendWith(MicrotusExtension.class)` (Cyano) and receives an `Env` parameter, from which it builds +instances with `env.createEmptyInstance(loader)` +([`AvesAnvilLoaderIntegrationTest`](../src/test/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoaderIntegrationTest.java)). +It needs a real server because the loader touches `Chunk`, `Section`, `Palette`, the block registry +and the biome registry. It covers the chunk round trip through a real region file: absent chunks, +block round trip, region file placement in the dimension directory, NBT on blocks, block properties, +parallel saving and parallel loading. The other two use the same extension: the lifecycle test covers +what a closed loader does with further loads and saves, the concurrency test the loader under several +threads at once, including the open-region limit and the eviction of a region file that is being read. + +There is no dedicated unit test for `BlockPaletteResolver` and `BiomePaletteResolver`; both are +covered only indirectly through the integration test. `BiomePaletteResolver` accepts an injectable +`Supplier>` for exactly this purpose +([`BiomePaletteResolver(AnvilDiagnostics, Supplier)`](../src/main/java/net/theevilreaper/aves/instance/anvil/BiomePaletteResolver.java)), +so a server-free test for it is possible and currently missing. + +Run everything with: + +```bash +./gradlew test +``` + +## References + +Minestom sources cited here, at version `2026.06.20-26.1.2`: + +* `net/minestom/server/instance/anvil/AnvilLoader.java` +* `net/minestom/server/instance/anvil/RegionFile.java` +* `net/minestom/server/instance/ChunkLoader.java` +* `net/minestom/server/instance/InstanceContainer.java` +* `net/minestom/server/instance/palette/PaletteImpl.java` +* `net/minestom/server/instance/palette/Palette.java` +* `net/minestom/server/utils/validate/Check.java` +* `net/minestom/server/instance/DynamicChunk.java` + +Format reference: [Region file format](https://minecraft.wiki/w/Region_file_format), +[Chunk format](https://minecraft.wiki/w/Chunk_format). diff --git a/docs/benchmarks.md b/docs/benchmarks.md new file mode 100644 index 00000000..f3569b22 --- /dev/null +++ b/docs/benchmarks.md @@ -0,0 +1,295 @@ +# Benchmarks + +JMH benchmarks for the [Anvil chunk loader](anvil-chunk-loader.md) and the +[light engine](light-engine.md). They live in their own source set, `src/jmh/java`, so nothing they +need ever reaches the classpath of the published library. + +> **Read this before quoting a number.** Everything a benchmark prints is a measurement of *one* +> machine, on *one* JVM, under *one* load. None of it belongs in `anvil-chunk-loader.md` or +> `light-engine.md` as an absolute statement. See [What the numbers are not](#what-the-numbers-are-not). + +## Running them + +```bash +./gradlew jmh # every benchmark, full settings +``` + +That is every measured method for every combination of the parameters its state class declares — 476 +configurations at the time of writing, at the forks and iterations the annotations ask for. Budget +well over an hour and do not touch the machine while it runs. The count is dominated by +`ScalingBenchmark`, whose fifteen section counts and five distinct-state counts share one state class +and therefore form a full cross product of 300 configurations on their own. + +The full run is not what you want during development. Restrict it to what you are working on: + +```bash +# One class +./gradlew jmh -Pjmh.include='BitPackerBenchmark' + +# One method +./gradlew jmh -Pjmh.include='ChunkSaveStageBenchmark.codec' + +# A regex over several +./gradlew jmh -Pjmh.include='light\..*Propagator.*' +``` + +The Gradle task writes two files: + +| File | Content | +| --- | --- | +| `build/reports/jmh/human.txt` | the console output | +| `build/reports/jmh/results.json` | machine readable, for [JMH Visualizer](https://jmh.morethan.io/) | + +### Running the jar directly + +The task builds a self-contained benchmark jar, which is the faster way to iterate because it skips +Gradle entirely and accepts every JMH option: + +```bash +./gradlew jmhJar +java -jar build/libs/aves-*-jmh.jar 'BitPackerBenchmark.pack' -f 1 -wi 1 -i 1 + +java -jar build/libs/aves-*-jmh.jar -l # list every benchmark +java -jar build/libs/aves-*-jmh.jar -h # every option +``` + +A quick smoke run — one fork, one warmup iteration, one measurement iteration — is `-f 1 -wi 1 -i 1`. +That is enough to prove a benchmark executes and produces a plausible number. It is **not** enough +to compare two versions of the code; for that, drop the overrides and let the annotations decide. + +### Profiling a benchmark + +```bash +java -jar build/libs/aves-*-jmh.jar 'PaletteDataBenchmark.encode' -prof gc +java -jar build/libs/aves-*-jmh.jar 'PaletteDataBenchmark.encode' -prof perfasm # Linux, needs perf +``` + +`-prof gc` is the one worth reaching for first. Several of the claims in the other two documents are +about *allocation*, not about time, and `gc.alloc.rate.norm` answers those directly. + +## Not part of `build` + +The benchmarks do not run during `./gradlew build`, `check` or `test`. `jmh` and `jmhJar` are only +reachable when asked for by name. A benchmark run takes long enough that wiring it into the normal +build would make every commit painful, and JMH numbers are too noisy on a shared CI runner to gate +anything on them anyway. + +Verify it for yourself: + +```bash +./gradlew clean build --dry-run | grep -i jmh # prints nothing except the compile task +``` + +`compileJmhJava` *is* part of `build`, and that is on purpose: a benchmark that no longer compiles +after a refactoring should break the build like any other source set. + +## Why no Minestom — and the benchmarks where the answer is different + +Two kinds of benchmark live here, and the distinction matters when reading a number. + +The **library benchmarks** — the large majority — start no server at all. They measure what this +code contributes, with the registry replaced by a fake. + +The **comparison benchmarks** measure this implementation against the one Minestom ships with, and +there the point *is* to run the original rather than a stand-in. Three of them need a server for it: + +| Benchmark | Server | Why | +| --- | --- | --- | +| `RegionFileComparisonBenchmark` | no | Minestom's `RegionFile` reads no registry, so it runs in a bare fork. The class sits in `net.minestom.server.instance.anvil` because that type is package-private. | +| `ChunkSaveComparisonBenchmark` | `MinecraftServer.init()` | Minestom's `AnvilLoader` reads the biome registry and the block state count in **static** fields, so the class initialiser fails before any measurement unless the registries exist. | +| `LightEngineComparisonBenchmark` | `MinecraftServer.init()` | Measures the original light engine, whose methods are package-private in `net.minestom.server.instance.light`. | +| `LightEngineStageBenchmark` | `MinecraftServer.init()` | Same package and the same reason: it splits both engines into their stages, so it calls the same package-private methods. | + +The comparison numbers therefore include what a real registry costs on both sides — which is +correct, because both sides pay it. They are not comparable with the library benchmarks below, which +deliberately exclude it. + +Everything from here on applies to the library benchmarks. + +Both packages already separate their algorithm from the registries of a running server — +`PaletteEntryResolver` for the codec and `BlockLightSource` for the light engine. The benchmarks +plug fakes into those two interfaces +([`FakePaletteEntryResolver`](../src/jmh/java/net/theevilreaper/aves/benchmark/support/FakePaletteEntryResolver.java), +[`FakeBlockLightSource`](../src/jmh/java/net/theevilreaper/aves/benchmark/support/FakeBlockLightSource.java)). + +This is a deliberate trade. A registry lookup is expensive enough to dominate every one of these +measurements, and a benchmark whose number is 90 % Minestom tells you nothing about the code in +this repository. The consequence is that **no benchmark here reports what a chunk load costs on a +real server.** They report what *this* library contributes to it. + +`FakeBlockLightSource` takes a `resolveCost` in [`Blackhole.consumeCPU`](https://javadoc.io/doc/org.openjdk.jmh/jmh-core/latest/org/openjdk/jmh/infra/Blackhole.html) +tokens for exactly this reason. `SectionOpacity` exists to resolve each distinct block state once +instead of once per visit, and how much that is worth depends entirely on what a resolution costs. +Measuring it against a free fake would make the cache look like pure overhead, which is the opposite +of what happens against a registry. + +## The benchmarks + +### Anvil + +| Benchmark | Parameters | What it answers | +| --- | --- | --- | +| [`BitPackerBenchmark`](../src/jmh/java/net/theevilreaper/aves/benchmark/anvil/BitPackerBenchmark.java) `.pack` `.unpack` `.roundTrip` | `bitsPerEntry` 4, 5, 8, 15 | What the packing loop of one section costs. It runs once per section on every load and every save, so a full-height chunk pays it 24 times. The parameter is the only thing that changes the iteration count per long. | +| [`PaletteDataBenchmark`](../src/jmh/java/net/theevilreaper/aves/benchmark/anvil/PaletteDataBenchmark.java) `.encode` `.unpack` `.roundTrip` | `distinctStates` 1, 8, 64, 200 | What collecting a palette costs as the section gets busier. `1` is a section of pure air or pure stone — the majority of every world — and is answered without packing anything. `200` already needs eight bits per entry. | +| [`ChunkCompressionBenchmark`](../src/jmh/java/net/theevilreaper/aves/benchmark/anvil/ChunkCompressionBenchmark.java) `.compress` `.decompress` | `compression` ZLIB, GZIP, NONE × `distinctStates` 8, 200 | The most expensive single stage of a chunk transfer, and the one the loader deliberately performs outside every lock. The payload is real serialised chunk NBT built through the same codec the save path uses, not random bytes — random bytes do not compress and would make zlib look far worse than it is. | +| [`RegionFileBenchmark`](../src/jmh/java/net/theevilreaper/aves/benchmark/anvil/RegionFileBenchmark.java) `.writeRaw` `.readRaw` `.roundTrip` | – | The byte transfer alone, on an already compressed payload. `readRaw` uses positional channel reads and takes no lock; `writeRaw` takes the region lock for the sector allocation and the header update. | +| [`ChunkSaveStageBenchmark`](../src/jmh/java/net/theevilreaper/aves/benchmark/anvil/ChunkSaveStageBenchmark.java) `.snapshot` `.codec` `.codecWithoutCompression` `.transfer` `.full` | `distinctStates` 8, 200 | **The interesting one.** See below. | + +`ChunkSaveStageBenchmark` splits a whole chunk save into the three stages it consists of, because +the central claim of the loader is structural and this is what turns it into a number: + +| Stage | Lock held | +| --- | --- | +| `snapshot` — copy the section arrays | the read lock of the chunk; a game thread waits here | +| `codec` — palettes, packing, NBT, compression | **none** | +| `transfer` — hand the finished bytes to the region file | the region lock, for the allocation and the header | +| `full` — all three, so the sum can be checked against the whole | – | + +`codecWithoutCompression` splits the middle stage again, into its palette half and its zlib half. + +The chunk is a [`ChunkColumn`](../src/jmh/java/net/theevilreaper/aves/benchmark/support/ChunkColumn.java) +of plain arrays, not a Minestom chunk. A real `Section` needs a started server, and the registry time +would land inside the `codec` stage and hide the very thing the benchmark isolates. + +### Light + +| Benchmark | Parameters | What it answers | +| --- | --- | --- | +| [`LightNibblesBenchmark`](../src/jmh/java/net/theevilreaper/aves/benchmark/light/LightNibblesBenchmark.java) `.getUniform` `.getAllocated` `.setUniformUnchanged` `.setAllocating` `.ofArray` | – | What the uniform shortcut is worth. A section whose blocks all carry the same level keeps no array and answers from a field; every other section shifts a nibble out of 2048 bytes. Most sections of a world are completely dark or completely sky-lit, so the shortcut is the common path, not the exception. | +| [`SectionOpacityBenchmark`](../src/jmh/java/net/theevilreaper/aves/benchmark/light/SectionOpacityBenchmark.java) `.of` | `distinctStates` 1, 8, 64, 200 × `resolveCost` 0, 50 | What "resolve each distinct state once" is worth. `distinctStates` sets how often the cache misses; `resolveCost` sets what a miss costs. At `resolveCost = 0` you are measuring the hash map and the two array writes per block, so the table looks like pure overhead. At `resolveCost = 50` you are measuring what it saves: seven resolutions per distinct state instead of seven per block. | +| [`LightPropagatorBenchmark`](../src/jmh/java/net/theevilreaper/aves/benchmark/light/LightPropagatorBenchmark.java) `.propagate` | `lightSources` 0, 1, 8, 64 × `occlusionPercent` 0, 25 | How the single-section search scales with the amount of queued positions. `lightSources = 0` is answered without a search at all, which is the case for the overwhelming majority of the sections of a world. `occlusionPercent = 25` is there because solid blocks stop the search early — measuring only an open section reports the worst case and calls it normal. | +| [`ChunkLightPropagatorBenchmark`](../src/jmh/java/net/theevilreaper/aves/benchmark/light/ChunkLightPropagatorBenchmark.java) `.propagate` `.propagateSky` | `sectionCount` 4, 16, 24 × `lightSourcesPerSection` 1, 8 | The same search across section borders, over a flat map (4), a shallow world (16) and a full-height overworld (24). Both searches are measured because the engine runs both per chunk and they behave very differently: block light is bounded by the amount of emitting blocks, sky light seeds nearly every block of an open column. | + +Both propagators keep their working buffers between runs, so the benchmarks reuse one instance for +the whole trial and warm the buffers in `@Setup`. A fresh instance per invocation would measure two +array allocations instead of the search. + +`ChunkLightState` and `ChunkLightService` have no benchmarks of their own. They are the +Minestom-facing half of the engine; what they cost is covered by the comparison benchmarks below, +which do start a server. + +### Against the implementations Minestom ships with + +These are the ones that answer "is this actually better", and they are the reason the +[loader](anvil-chunk-loader.md) and [light engine](light-engine.md) documents can state factors +instead of intentions. Each measures the original, not a reimplementation of it — which is why three +of them live in Minestom packages, where the measured types are package-private. + +| Benchmark | Parameters | What it answers | +| --- | --- | --- | +| [`RegionFileComparisonBenchmark`](../src/jmh/java/net/minestom/server/instance/anvil/RegionFileComparisonBenchmark.java) `.avesRead` `.minestomRead` `.avesWrite` `.minestomWrite` | `distinctStates` 8, 200, and the JMH thread count | The central claim of the loader: what the lock granularity is worth. Run it with `-t 1` and the two are level; the difference appears only under contention, which is why the thread count is the parameter that matters here. | +| [`ChunkSaveComparisonBenchmark`](../src/jmh/java/net/theevilreaper/aves/benchmark/anvil/ChunkSaveComparisonBenchmark.java) `.avesSave` `.minestomSave` `.compressAvesLevel` `.compressMinestomLevel` | `distinctStates` 1, 16, 64, 256, 1024 | Whether the palette handling shows up in a whole chunk save. Both sides run the identical Adventure writer over byte-identical payloads, so the save comparison cannot be won by choosing a cheaper compression level — that variable is measured separately in the two `compress*` methods. | +| [`LightEngineComparisonBenchmark`](../src/jmh/java/net/minestom/server/instance/light/LightEngineComparisonBenchmark.java) `.aves` `.minestom` | `lightSources` 1, 8, 64 × `occlusionPercent` 0, 30 × `emissionMix` UNIFORM, MIXED | By how much each light engine wins, and on what shape of section. All three parameters are needed: the margin moves with each of them. | +| [`LightEngineStageBenchmark`](../src/jmh/java/net/minestom/server/instance/light/LightEngineStageBenchmark.java) `.avesReadStates` `.avesOpacity` `.avesPropagate` `.avesCollect` `.avesFull` `.minestomQueue` `.minestomFull` | `lightSources` 1, 8, 64 × `occlusionPercent` 0, 30 | *Why* one of them wins, which the comparison never says. It splits the Aves path into reading the palette, building the opacity table, searching and packing, and the built-in path into building the seed queue and the rest. This is what identified the allocation the opacity table used to make, and it is the source of the stage table in [`light-engine.md`](light-engine.md#where-the-gain-came-from). | + +`emissionMix` decides whether every source of the section emits level 15 (`UNIFORM`, glowstone +throughout) or whether the sources differ (`MIXED`: glowstone 15, lantern 15, torch 14, redstone +torch 7, magma block 3, at the same positions drawn from the same seed). The Aves search assumes its +queued positions are ordered by level, which only holds while every source starts at the same one, so +this parameter is what would show whether a bucket queue is worth adding. One cell of the cross +product measures nothing: with a single source `MIXED` places glowstone as well and is a duplicate of +`UNIFORM`. The recommended run therefore leaves it out: + +```bash +java -jar build/libs/aves-*-jmh.jar "LightEngineComparisonBenchmark.(aves|minestom)" \ + -p emissionMix=UNIFORM -f 1 -wi 3 -i 5 +java -jar build/libs/aves-*-jmh.jar "LightEngineComparisonBenchmark.(aves|minestom)" \ + -p emissionMix=MIXED -p lightSources=8,64 -f 1 -wi 3 -i 5 +``` + +**The comparison verifies the two engines agree before it measures them.** Its `@Setup` runs both +paths over the section it just built and aborts the trial when the 2048 bytes differ, so a faster +number cannot come from computing something else. `LightEngineEquivalenceTest` pins the same property +down in the normal test run, over 54 scenarios. Both are recent: the byte identity was stated in +[`light-engine.md`](light-engine.md) long before anything in the build checked it. + +Because three of them start a server, their absolute numbers include registry time and are **not** +comparable with the library benchmarks above. Compare them only against their own counterpart. + +### Scaling beyond the sizes anyone uses + +[`ScalingBenchmark`](../src/jmh/java/net/theevilreaper/aves/benchmark/ScalingBenchmark.java) measures +this library against itself rather than against Minestom, along two axes that the other benchmarks +sample too coarsely to expose a bend in: + +| Method | Parameter | What it answers | +| --- | --- | --- | +| `blockLightBySectionCount` `skyLightBySectionCount` | `sectionCount` 1 … 256, fifteen steps | Whether cost per section stays flat as the world grows taller. Block light does across the whole range. Sky light does not: a least-squares fit over the vanilla range (≤ 24 sections) understates the measured cost at 256 sections by about 20 %, while the same method lands within 2 % for block light. | +| `paletteByDistinctStates` `packingByDistinctStates` | `distinctStates` 1 … 1024 | The same question for the codec as a section fills up. | + +Fifteen steps rather than three, because the point of this benchmark is to find where a curve stops +being straight — and that is exactly what a coarse parameter set hides. + +## Benchmark hygiene + +The rules every benchmark in this source set follows: + +- **Inputs are built in `@Setup`, never in the measured method.** Otherwise the generator is what + gets measured. +- **Every measured method returns its result** so the JIT cannot delete the work. Where there is no + natural result — `RegionFileBenchmark.writeRaw`, `ChunkSaveStageBenchmark.transfer` — the side + effect on the file is what keeps it alive. +- **`@Fork`, `@Warmup` and `@Measurement` are explicit on every class.** JMH's defaults are fine but + invisible, and a reader should be able to see how much evidence a number rests on without + consulting the JMH manual. +- **`@BenchmarkMode` and `@OutputTimeUnit` are set per class.** Everything here is `AverageTime` in + microseconds, because every operation is a whole-section or whole-chunk unit of work and + throughput would be the less natural framing. +- **`@State(Scope.Thread)`** everywhere. None of the measured code is designed to be shared between + threads: `LightPropagator` and `ChunkLightPropagator` are explicitly documented as single-thread + confined, and the state classes hold mutable buffers. +- **Sweeps, not single accesses.** `LightNibblesBenchmark` walks all 4096 blocks per invocation + instead of reading one nibble. A single nibble read is a handful of instructions, which is below + what a harness can separate from its own overhead. Divide by 4096 for the per-access cost — but + see the caveat below. +- **Deterministic inputs.** Every generator seeds from `BenchmarkConstants.SEED`. Two runs of the + same benchmark must see byte-identical input, or the difference between them describes the input + and not the change. + +### The one place a sweep is still eliminated + +`LightNibblesBenchmark.getUniform` and `.setUniformUnchanged` report times far below one nanosecond +per access. That is not a measurement error and no `Blackhole` fixes it — verified by rerunning with +`-Djmh.blackhole.autoDetect=false`, which produces the same number. + +A uniform section answers every read from a single field, whatever the coordinates. The loop body is +therefore loop-invariant, the JIT hoists it out, and the empty loop disappears. `setUniformUnchanged` +goes the same way: writing the level the section already carries returns immediately, the compiler +proves the loop has no effect, and removes it. + +These two numbers are **lower bounds, not per-access costs.** What they legitimately say is that the +uniform path can collapse to a single field read — which is precisely the property the shortcut +exists for. Only `getAllocated` and `setAllocating` divide meaningfully by 4096. Both are kept +because a change that accidentally made the uniform path allocate would show up here instantly. + +## What the numbers are not + +- **They are not portable.** Core count, CPU frequency scaling, the JIT's mood on the day, the page + cache and the file system all move these numbers. A result from your laptop says nothing about a + production host. +- **They are not a chunk load time.** No benchmark here includes registry lookups, chunk allocation, + or anything else Minestom does around this code. See [Why no Minestom](#why-no-minestom). +- **`RegionFileBenchmark` is not a storage benchmark.** It runs on a warm page cache and measures + almost no device time. That is realistic for a server saving the same chunks repeatedly, and + useless as a statement about a disk. +- **A single `-f 1 -wi 1 -i 1` run proves nothing.** It proves the benchmark executes. Any + comparison needs the configured forks and iterations, and preferably `-prof gc` alongside. + +If a number from here ends up in `anvil-chunk-loader.md` or `light-engine.md`, it needs the machine, +the JVM and the JMH configuration next to it, or it is not a fact — it is a rumour with a decimal +point. + +## Notes on the setup + +| Piece | Version | Why | +| --- | --- | --- | +| [`me.champeau.jmh`](https://github.com/melix/jmh-gradle-plugin) | `0.7.3` | Latest release. Gives the benchmarks their own source set so JMH never lands on the main or test classpath of a published library. | +| `org.openjdk.jmh:jmh-core` | `1.37` | Latest release. **Not managed by `mycelium-bom`** — the BOM covers adventure, minestom, cyano, junit and mockito only — so both versions are pinned explicitly in `settings.gradle.kts`. | +| `net.kyori:adventure-bom` | `5.1.1` | The main source set gets adventure through `compileOnly(minestom)`, which never reaches a runtime classpath. The benchmarks run their code for real and need adventure at runtime, so they import the platform directly. **Keep this in sync with the version Minestom resolves to** — check with `./gradlew dependencyInsight --configuration compileClasspath --dependency adventure-nbt`. | + +The plugin generates the harness classes with its bytecode generator. No `jmhAnnotationProcessor` is +declared on purpose: with both present, the two generators emit the same classes and the jar ends up +with two copies of every benchmark. + +On Java 25, JMH 1.37 prints a warning about `sun.misc.Unsafe::objectFieldOffset` being terminally +deprecated. It is harmless and comes from JMH itself, not from this repository. diff --git a/docs/light-engine.md b/docs/light-engine.md new file mode 100644 index 00000000..b243588b --- /dev/null +++ b/docs/light-engine.md @@ -0,0 +1,556 @@ +# Light engine + +A light propagation for a single section that resolves the properties of a block once instead of on +every visit, and stores a uniform section without an array. The algorithm itself knows nothing about +Minestom, which is what makes it testable without a running server. + +> **Experimental.** Every public type of `net.theevilreaper.aves.instance.light` is annotated +> `@ApiStatus.Experimental`. The API may still change. + +> **Scope.** Block light and sky light for a chunk, across section borders, across chunk borders, +> and incrementally after a single block changed. Works with any chunk of any loader. +> See [Limits](#limits) for what is still missing. + +## When this is worth using + +Honest answer first, because it decides whether the package is relevant at all: + +| Workload | Does light computation run? | +| --- | --- | +| Loading pre-lit worlds from `.mca` | **No.** The stored light is applied with `Light#set`, which clears `requiresUpdate()`. Nothing is recomputed — unless you call this engine explicitly. | +| Generated worlds without stored light | Yes | +| Runtime block placement | Yes | + +For loading pre-built maps from region files — the dominant Aves use case — a light engine +contributes nothing, because that code path never executes. The measured cost of loading such a +chunk is dominated by NBT parsing and zlib inflation instead. + +## Design + +Seven types, each with one responsibility. Only the two on the right know Minestom exists. + +``` + ┌─ engine, no Minestom ─────────────┐ ┌─ adapter ──────────────┐ + │ │ │ │ + Chunk ──────────────┼──► int[] stateIds │ │ ChunkLightService │ + │ │ │ │ │ │ + │ ▼ │ │ │ uses │ + │ SectionOpacity ◄── BlockLightSource ◄── MinestomBlockLight- │ + │ │ (one lookup │ │ Source │ + │ ▼ per state) │ │ │ + │ ChunkLightPropagator │ │ writes back through │ + │ │ (crosses section │ │ Light#set(byte[]) │ + │ ▼ borders) │ │ │ + │ List ──────────────┼───┼──► Chunk sections │ + └───────────────────────────────────┘ └────────────────────────┘ +``` + +| Type | Responsibility | +| --- | --- | +| `LightNibbles` | Storage. Two levels per byte, uniform sections without an array. | +| `BlockLightSource` | Abstraction over "how bright is this block and which faces does it block". | +| `SectionOpacity` | Precomputed table of those properties for one section. | +| `LightPropagator` | The breadth-first propagation, with reusable buffers. | +| `ChunkLightPropagator` | The same search across all sections of a chunk, so light crosses their borders. | +| `MinestomBlockLightSource` | Answers `BlockLightSource` from the block registry. | +| `ChunkLightState` | Keeps a calculated result and updates it incrementally, including the retraction pass and the sky heightmap. | +| `ChunkLightService` | Reads a chunk, runs the propagation, settles the borders against the neighbours, writes the result back. | + +`BlockLightSource` exists for the same reason `PaletteEntryResolver` does in the Anvil package: it +keeps the registry out of the algorithm, so the propagation is verified with a handful of fake +blocks and no server at all. + +## Where the resources are saved + +### Time + +The dominant cost of a naive propagation is not the search but the block lookups. A breadth-first +search reaches a block from up to six directions, and resolving palette → block → registry → +occlusion shape on each of those visits repeats the same work. + +`SectionOpacity` resolves every **distinct state id** of a section exactly once when the table is +built and answers from two flat `byte[]` afterwards — an array index instead of a registry walk. +A section of 4096 stone blocks costs one lookup, which +`SectionOpacityTest#testEveryDistinctStateIsResolvedOnlyOnce` pins down. + +Two further short cuts: + +- A section without any emitting block returns immediately with a uniform dark result. No buffer is + touched, no queue is built. +- Because a level drops by exactly one per block and the search is breadth-first, every position is + reached with its final level on the first visit. No position is ever revisited or re-queued. + +### Memory + +- **Uniform sections carry no array.** `LightNibbles` keeps a single level and allocates the + 2048-byte array only when a level actually differs. Most sections of a world are either fully dark + or fully lit, so this is the common case rather than an edge case. `fill` releases the array again. +- **A fully dark section reports an empty array** (`toArray().length == 0`), which is how the file + format stores "no light" — nothing is written for it. +- **The propagator reuses its buffers.** The level buffer and the queue are allocated once per + instance and cleared per run, so repeated propagation allocates nothing beyond the result. An + instance is therefore reusable but thread confined — use one per worker rather than sharing one. +- **The queue is an `int[]`, not a collection.** No boxing, no growth: a section has 4096 positions + and each is queued at most once, so the array is sized exactly once. +- **Building the table allocates nothing per block.** The lookup is a linear probing table over the + raw state id, so no key is boxed and no value object is created; a resolved state is packed into a + short that carries the occluded faces and the emission together. A run of one repeated state is + answered from the previous block instead of the table, because the blocks of a world come in runs. + This is what took the build of one section from 74 040 to 8 664 bytes and is the single largest + reason for the times [further down](#compared-with-the-light-engine-minestom-ships-with). + +## Correctness details worth knowing + +**Occlusion is per face, not per block.** Roughly one in seven block types of the game occludes some +faces and not others — slabs, stairs, snow, farmland, dirt paths, lecterns, stonecutters. A design +storing one flag per block answers those wrongly. `SectionOpacity` stores a six-bit mask per block; +`MinestomBlockLightSourceTest` pins the behaviour down on real bottom and top slabs. + +**Only the entered face is tested.** Light passing from A to B is blocked by the face of **B** it +enters, not by the face of A it leaves. Testing both would leave every emitting block that is opaque +itself dark — and glowstone is exactly that. This is checked end to end against the real registry. + +**Unknown block states are transparent, not fatal.** `Block.fromStateId` indexes an array without a +bounds check and throws for an id outside the known range. The adapter turns that into an absent +block, because a propagation must not lose a whole section over one unknown state. + +**The face mapping is pinned by a test.** The adapter maps its faces onto the server's by ordinal. +`testTheFaceOrderMatchesTheOneOfTheServer` fails if Minestom ever reorders its enum, which would +otherwise silently shift every occlusion answer to the wrong face. + +## Compared with the light engine Minestom ships with + +`LightEngineComparisonBenchmark` runs both engines over the same section, from a block palette to a +finished light array of 2048 bytes. It lives in `net.minestom.server.instance.light` because the two +methods that make up the built-in path — `BlockLight.buildInternalQueue` and `LightCompute.compute` — +are package-private, which is the only way to measure the original instead of a copy of it. Neither +side gets to skip its preparation: the built-in path builds its seed queue, and the Aves path builds +its opacity table through the real block registry rather than a stand-in. + +**The two engines produce the same light, and that is now checked rather than claimed.** +`LightEngineEquivalenceTest` compares them byte for byte over 54 scenarios — nine source counts +against six shares of solid blocks — and runs with `./gradlew test` like any other test. The +benchmark repeats the same check in its `@Setup` and aborts the trial when the two disagree. Until +recently this document asserted the byte identity while nothing in the build verified it; the +statement happened to be true, but a change that broke it would have passed unnoticed. Nothing in +this section is a statement about correctness — correctness is equal, not better. Everything below is +about time. + +The numbers in this section were measured with `-f 1 -wi 5 -i 10` on a quiet machine, one section per +operation, `score ± error`, lower is better. Every source emits level 15; sections whose sources +differ in brightness are measured separately, [further down](#sources-of-mixed-brightness). + +``` +java -jar build/libs/aves-*-jmh.jar "LightEngineComparisonBenchmark.(aves|minestom)" \ + -p emissionMix=UNIFORM -f 1 -wi 5 -i 10 +``` + +| Light sources | Solid blocks | Aves | Minestom | Aves is | +| ---: | ---: | ---: | ---: | --- | +| 1 | 0 % | 44.5 ± 0.6 µs/op | 49.4 ± 1.3 µs/op | 1.11× faster | +| 8 | 0 % | 98.3 ± 2.4 µs/op | 121.1 ± 5.5 µs/op | 1.23× faster | +| 64 | 0 % | 109.2 ± 1.6 µs/op | 126.5 ± 5.6 µs/op | 1.16× faster | +| 1 | 30 % | 39.3 ± 0.8 µs/op | 62.0 ± 2.0 µs/op | 1.58× faster | +| 8 | 30 % | 119.3 ± 3.5 µs/op | 204.2 ± 3.7 µs/op | 1.71× faster | +| 64 | 30 % | 122.6 ± 1.3 µs/op | 206.6 ± 4.2 µs/op | 1.68× faster | + +These numbers replace an earlier set in which Aves lost four of the six scenarios. What changed is +not the algorithm but the table it works from: building `SectionOpacity` allocated a throwaway lambda +per block, which is [broken down below](#where-the-gain-came-from). A shorter run on a loaded machine +reproduces the same ordering and the same rough magnitudes, with the wider spreads a loaded machine +produces. + +### A section without solid blocks: the narrow half of the result + +This is where the two engines are closest, because it is where Aves has the least to gain: nothing +blocks the light, so the search rarely has to ask whether it may pass. + +```mermaid +%%{init: {"themeVariables": {"xyChart": {"plotColorPalette": "#56B4E9, #E69F00"}}}}%% +xychart-beta + title "No solid blocks: Aves (blue, lower) against Minestom (orange, upper)" + x-axis "Light sources in the section" [1, 8, 64] + y-axis "Microseconds per section, lower is better" 0 --> 220 + line [44.5, 98.3, 109.2] + line [49.4, 121.1, 126.5] +``` + +`xychart-beta` draws no legend, so: the first line, the lower one, is **Aves** (blue); the upper one +is **Minestom** (orange). The scale runs to 220 although nothing here comes close to it, so that this +chart and the next one can be held against each other. + +**Why the margin is small here.** Before Aves computes anything, it goes through the section once and +notes down for every block whether light passes through it. That note costs time before a single ray +has moved. In a section with nothing in it, the search hardly ever consults it, so the preparation is +paid for and barely used. This is the shape of the workload on which Aves was behind until the +preparation itself became cheap enough for the remainder to be earned back; 1.11× at one source is +what is left of that, and it is the smallest margin in the table for exactly this reason. + +**How firm that is.** The spreads do not overlap at any of the three points — 44.5 ± 0.6 against +49.4 ± 1.3, 98.3 ± 2.4 against 121.1 ± 5.5, 109.2 ± 1.6 against 126.5 ± 5.6. The direction is +established; the size of the gap at one source is small enough that a different machine could move it. + +### A section with solid blocks: the wide half + +Once 30 % of the blocks are solid, the same mechanism works in the other direction. + +```mermaid +%%{init: {"themeVariables": {"xyChart": {"plotColorPalette": "#56B4E9, #E69F00"}}}}%% +xychart-beta + title "30 percent solid blocks: Aves (blue, lower) against Minestom (orange, upper)" + x-axis "Light sources in the section" [1, 8, 64] + y-axis "Microseconds per section, lower is better" 0 --> 220 + line [39.3, 119.3, 122.6] + line [62.0, 204.2, 206.6] +``` + +Same order and the same colours as before: the first line is **Aves** (blue), the second **Minestom** +(orange). The distance between the lines is several times what the previous chart shows, on the same +scale. + +**Why.** Now the note earns its keep. Solid blocks are exactly what a spreading light keeps running +into, and every time it does, the same question comes up again: does light get through here? Aves +reads the answer off the note it wrote at the start — one position in an array. Minestom asks the +registry again each time. The more often the question is asked, the more the one-off cost of writing +the note is worth; and how often it is asked is set by how many light sources are spreading and how +much they run into. + +That is the whole shape of the result, and it is what the optimisation moved: + +```mermaid +flowchart TB + AV["Aves
pays once up front: one registry lookup per
distinct block state of the section
then one array read per question"] + MI["Minestom
pays nothing up front
then one registry lookup per question"] + Q{"How many times does the search ask
'does light pass through here?'"} + AV --> Q + MI --> Q + Q -->|"few times: nearly empty section,
the search runs out quickly"| L["the up-front cost is barely used.
Since it became cheap it is still
earned back, but only just: 1.11x to 1.23x"] + Q -->|"many times: solid blocks everywhere,
every step runs into one"| W["the cheap answers add up
1.58x to 1.71x"] +``` + +The up-front cost has not disappeared, it has shrunk. The break-even point that used to sit inside +the measured range now sits below its sparsest configuration, which is why the left branch reads as a +small win instead of a loss. A section that is sparser still — no sources at all — is answered +without a search or a table at all, so it never reaches this decision. + +**How firm that is.** None of the six spreads overlap. The widest margins, 8 and 64 sources at 30 % +solid, are also the ones with the tightest errors on both sides: 119.3 ± 3.5 against 204.2 ± 3.7 and +122.6 ± 1.3 against 206.6 ± 4.2. + +### Where the gain came from + +`LightEngineStageBenchmark` splits both engines into their stages, which is what identifies the cost. +For one light source in an open section, in microseconds: + +| | readStates | opacity | propagate | collect | full | +| --- | ---: | ---: | ---: | ---: | ---: | +| before | 7.70 | **31.33** | 33.85 | 0.24 | 77.1 | +| after | 7.23 | **8.07** | 31.41 | 0.23 | 46.3 | + +Building the opacity table was 41 % of the whole path and is now 17 % of a much shorter one. The +table allocated a throwaway lambda per block; removing that took the allocation of one call from +74 040 bytes to 8 664. The search and the two transfer stages are unchanged, as they should be — +nothing about the algorithm was touched. + +**One number in there corrects the story this document used to tell.** The Aves *search* was already +the faster of the two before the change: 33.9 µs against the 53.9 µs the built-in search takes. The +entire deficit came from the preparation, not from the propagation. The earlier text explained the +losses as a property of the algorithm's shape; they were a property of one allocation. + +### Sources of mixed brightness + +Every scenario above places glowstone, so every source starts at level 15. Real interiors are lit +with torches, lanterns and magma blocks side by side, and `emissionMix=MIXED` builds exactly that: +the same positions, drawn from the same seed, filled with glowstone 15, lantern 15, torch 14, +redstone torch 7 and magma block 3. + +``` +java -jar build/libs/aves-*-jmh.jar "LightEngineComparisonBenchmark.(aves|minestom)" \ + -p emissionMix=MIXED -p lightSources=8,64 -f 1 -wi 3 -i 5 +``` + +| Light sources | Solid blocks | Aves | Minestom | Aves is | +| ---: | ---: | ---: | ---: | --- | +| 8 | 0 % | 118.97 ± 8.89 µs/op | 126.54 ± 9.55 µs/op | 1.06× faster | +| 8 | 30 % | 116.50 ± 7.63 µs/op | 201.46 ± 16.83 µs/op | 1.73× faster | +| 64 | 0 % | 150.42 ± 32.73 µs/op | 162.20 ± 3.11 µs/op | 1.08× faster | +| 64 | 30 % | 149.42 ± 12.64 µs/op | 252.26 ± 9.28 µs/op | 1.69× faster | + +A single source is not measured under `MIXED`: the first block of the set is glowstone, so a lone +source is the identical section `UNIFORM` already covers. + +**Mixed levels cost Aves more than they cost Minestom.** Held against the same scenario under +`UNIFORM` in the same run, the Aves time at 64 sources in an open section rises by about a third, and +the margin over Minestom falls from roughly 1.3× to the 1.06× above — the mixture costs Aves more +than it costs the built-in engine. The reason is in the search: it assumes the queued positions are +ordered by level, which is true only while every source starts at the same one. With mixed levels a +position can be reached again later by a brighter wave, and the same positions are touched more than +once. In the 30 % rows the margin is unaffected — 1.73× and 1.69× against the 1.71× and 1.68× of the +uniform table — because there the cheap opacity answers still dominate what the search spends. + +Aves is ahead in all four, but the open-section pair is close enough that the two are effectively +level there. If a bucket queue is ever added to the propagator, this is the parameter that will show +whether it was worth it — and the reason it is a parameter rather than a constant. + +### Which of the two is the steadier + +In the run above, Aves has the smaller absolute spread at every one of the six points: ± 0.6 to +± 3.5 against ± 1.3 to ± 5.6. Measured relative to the score the picture is almost the same, with one +exception — at 8 sources and 30 % solid Minestom is the tighter of the two (1.8 % against 2.9 %). +The earlier version of this document reported the opposite across the board, on an earlier run and a +loaded machine; the drop in allocation is the likeliest reason the Aves spread fell, since it removes +the garbage collector from the measurement. + +That comparison holds within one run, on one machine. The confirmation run on a loaded machine has +spreads several times wider on both sides, which is what a loaded machine does and not a property of +either engine. + +### On concurrency there is nothing to win here + +The Anvil comparison in [`anvil-chunk-loader.md`](anvil-chunk-loader.md) turns on a lock that is held +across expensive work. It would be convenient to claim the same thing on the light side, and it is +not true. Minestom's light path is already built for several threads: `LightCompute` is purely static +and allocates its buffer per call, `BlockLight` keeps its buffers per section, and `LightingChunk` +already uses an `Executors.newWorkStealingPool()`. Nothing in there serialises work that could be +running in parallel, so there is no contention to remove. + +### How the light reaches the chunk + +This is the one argument for this engine that does not depend on a measurement, and it is the +strongest one. Minestom computes light **only** inside `LightingChunk` (`LightingChunk extends +DynamicChunk`). Use any other chunk implementation and no light is computed at all. Aves computes +outside the chunk and hands the finished array over through `Light#set`, which every chunk accepts. + +```mermaid +flowchart TB + subgraph mine["Minestom: the light lives inside one chunk class"] + direction TB + M1["LightingChunk
(extends DynamicChunk)"] --> M2["computes its own light internally"] + M2 --> M3["sections are lit"] + M4["any other Chunk implementation"] --> M5["no light at all"] + end + subgraph aves["Aves: the light is computed outside and handed in"] + direction TB + A1["any Chunk — LightingChunk,
DynamicChunk, your own"] --> A2["ChunkLightService reads the block states"] + A2 --> A3["propagation runs outside the chunk,
knowing nothing about Minestom"] + A3 --> A4["Light#set(byte[]) per section"] + A4 --> A5["sections are lit"] + end +``` + +The same property has a second consequence: because the propagation references no Minestom class at +all, it can be tested against a handful of fake blocks without a running server. Only the adapter +that answers from the real registry needs one. + +### When to use Minestom's engine instead + +The reason this section used to give — the built-in engine is faster on sparsely occupied sections — +no longer holds, so it needs restating rather than deleting. + +If you already use `LightingChunk`, there is still no obligation to change anything. The built-in +engine is wired into the server, costs no extra code and no extra call site, and on an open section +lit by sources of differing brightness the two are within a few percent of each other. Swapping a +working light path for a margin that small is not a good trade on its own. + +What does argue for this engine is the case the built-in one does not cover at all: any chunk that is +not a `LightingChunk`, which includes the chunk type an `InstanceContainer` uses unless it is told +otherwise. After that come the workloads where the margin is actually large — sections carrying a +real share of solid blocks, where the measurements above put it between 1.58× and 1.73× — and the +control over *when* light is computed that comes from computing it outside the chunk. + +## Usage + +```java +import net.theevilreaper.aves.instance.light.LightNibbles; +import net.theevilreaper.aves.instance.light.LightPropagator; +import net.theevilreaper.aves.instance.light.MinestomBlockLightSource; +import net.theevilreaper.aves.instance.light.SectionOpacity; + +// One per worker thread; it keeps reusable buffers. +LightPropagator propagator = new LightPropagator(); +MinestomBlockLightSource source = new MinestomBlockLightSource(); + +int[] stateIds = new int[LightNibbles.BLOCK_COUNT]; // block states of one section +// ... fill stateIds from a palette ... + +LightNibbles light = propagator.propagate(SectionOpacity.of(stateIds, source)); + +int level = light.get(8, 8, 8); +byte[] stored = light.toArray(); // empty when the section is dark +``` + +`BlockLightSource` can be implemented directly to run the engine without a server: + +```java +BlockLightSource fake = new BlockLightSource() { + @Override public int emission(int stateId) { return stateId == LAMP ? 15 : 0; } + @Override public boolean blocksFace(int stateId, BlockFace face) { return stateId == STONE; } +}; +``` + +## Using it with a chunk loader or an instance + +`ChunkLightService` is the entry point. It reads the block states of a chunk, propagates, and hands +the result to the sections through `Light#set(byte[])`: + +```java +import net.theevilreaper.aves.instance.light.ChunkLightService; + +ChunkLightService lighting = new ChunkLightService(); // one per worker thread + +Chunk chunk = instance.loadChunk(0, 0).join(); +lighting.calculate(chunk); + +int level = lighting.blockLightAt(chunk, 8, 40, 8); +``` + +This works with **any** chunk, regardless of which loader produced it — the Anvil loader of Aves, +the one Minestom ships with, or a generated chunk. A test covers the round trip through +`AvesAnvilLoader` explicitly. + +Two properties make this the stable way in: + +- `Light#set(byte[])` is not marked internal, unlike `calculateInternal` / `calculateExternal`. The + service therefore does not implement the `Light` interface and cannot break when the signatures of + those internal methods change. +- `set` clears the update flag of the section, so the server does not recompute what was just + written. + +Locking follows the same three-stage split the Anvil loader uses: block states are read under the +read lock, the propagation runs with **no** lock held, and only the transfer of the result takes the +write lock. + +### Sky light + +```java +lighting.calculateSky(chunk); +``` + +Sky light enters from above and falls straight down **without losing a level** until something stops +it — which is why an open field is fully lit at every height while a cave is dark. Only after the +fall is interrupted does it spread like any other light, losing one level per block. + +### Across chunk borders + +```java +lighting.calculateWithNeighbours(instance, chunkX, chunkZ); +``` + +Lighting a chunk on its own ends its light at the border, which shows up as a straight dark line +every sixteen blocks. This method exchanges the border levels with every already loaded neighbour in +both directions. Neighbours that are not loaded are skipped rather than forced to load. + +One round of that exchange is not enough. A source in the corner of a chunk sends light through two +borders, and the light that entered a neighbour has to leave it again on another side to arrive in +the chunk diagonally behind it. The exchange therefore repeats over the whole area — the chunk and +the eight positions around it — until no chunk of it raises a level any more: + +| Property | How it is reached | +| --- | --- | +| Terminates | An injection only ever raises a level, and a level is capped at fifteen, so the repetition walks towards a fixed point. | +| Same result every time | The area is a fixed-size array walked in a fixed order, not a map. Since every step only raises levels, the fixed point does not depend on the order either. | +| Reads a chunk once | The opacity tables of every participating chunk are built once, before the first round, and reused by all of them. | +| Cannot loop forever | The amount of rounds is capped at sixteen, which is one more than the highest level that can exist. Hitting the cap is reported through `LOGGER.warn` instead of being accepted silently. | + +A radius of one chunk is enough because a level of fifteen cannot survive sixteen blocks of travel, +so nothing the middle chunk emits can reach a second ring. + +### Incremental updates + +`ChunkLightService#calculate` always recomputes the whole chunk. For a single block change that is +wasteful, and `ChunkLightState` exists for that case: + +```java +ChunkLightState state = ChunkLightState.blockLight(opacityTables); + +// after a block changed at that position +state.update(updatedOpacityTables, x, y, z); +List light = state.toSections(); +``` + +Adding brightness is straightforward — it only spreads. **Removing** it is the hard case and the +reason this class exists: when a light source disappears, the brightness it had spread is still +stored in every block around it, and spreading again would keep that glow forever. The update +therefore runs two passes. The first retracts every level that originated from the changed position +and collects the still valid levels it meets at the edge of the retracted area; the second spreads +those back in. + +Which way an update goes, drawn out: + +```mermaid +flowchart TB + C["a block changed at x, y, z"] --> K{"is the position
brighter or darker than before?"} + K -->|brighter| S["one pass: spread outwards.
Levels only ever rise, so nothing
has to be taken back"] + K -->|darker| R1["pass 1: retract.
Walk outwards and clear every level
that came from this position"] + R1 --> R2["at the edge of the cleared area,
collect the levels that came
from somewhere else"] + R2 --> R3["pass 2: spread those collected
levels back in"] + S --> D["result is identical to
a full recalculation"] + R3 --> D +``` + +The second pass is not a correction of the first. The light of every *other* source in the +neighbourhood is legitimate and was cleared along with the rest simply because it stood in the way; +collecting it at the edge and letting it back in is what puts it back. Skipping the retraction +instead and only spreading again would leave the removed source's glow in place for good. + +`ChunkLightStateTest#testTheIncrementalResultMatchesAFullRecalculation` asserts that the incremental +result is identical to a full recalculation, block for block. + +#### Sky light updates + +Sky light has an origin no block holds: it falls in from above. An update can therefore not tell +from the levels alone which positions lost their origin and which gained one, and a state that holds +sky light keeps a heightmap for that reason — the highest position that stops the sky, per column. + +A block change moves exactly one column of that heightmap, and the difference between the old and +the new height names the positions whose origin changed: + +| Change | Effect on the column | +| --- | --- | +| A block is placed above the current height | Everything between the old and the new height falls out of the open sky and gives its level back. What is left is refilled from the sides, which is why a single pillar leaves a level of fourteen below it rather than darkness. | +| The highest blocking block is removed | The column opens down to the next block below, and every position in between receives the full level again and spreads it. | +| The change is below the height | The height stays where it is. The changed position is retracted and refilled from its neighbours, exactly as a block light update works. | + +Only the changed column is walked again, so an update no longer re-seeds all two hundred and fifty +six columns of the chunk. + +`SkyLightUpdateTest` asserts the result against a full recalculation block for block, for both +directions, for a change that is not in the highest blocking position, and for a seeded sequence of +random changes that verifies the equality after every single one of them. + +### When to call what + +| Situation | Method | +| --- | --- | +| Chunk loaded without stored light, or generated | `calculate` / `calculateSky` | +| Chunk loaded and neighbours matter | `calculateWithNeighbours` | +| A single block changed | `ChunkLightState#update` | + +## Limits + +- **No `Light` implementation.** The engine deliberately does not implement + `net.minestom.server.instance.light.Light`. It writes its result through `set` instead, which + avoids depending on the internal calculation methods of that interface. +- **The exchange covers one ring of chunks.** `calculateWithNeighbours` settles the chunk and the + eight positions around it. That is enough for the light of the middle chunk, but the outer chunks + of the area are not settled against their own neighbours outside of it. +- **`Section.clone()` discards foreign light.** Should an adapter be built later, note that + `Section.clone()` calls `Light.sky()` / `Light.block()` outright, so any custom implementation is + silently replaced on copy. `LightingChunk.copy()` would have to be overridden. + +## Tests + +Everything that tests the algorithm itself runs without a Minestom server. Four classes need one and +use Cyano's `MicrotusExtension` for it, each because a server is what the test is about: + +| Class | Why it needs a server | +| --- | --- | +| `MinestomBlockLightSourceTest` | The directional occlusion of a slab is only meaningful against real block data. | +| `LightEngineEquivalenceTest` | Compares the two engines byte for byte over 54 scenarios; an equivalence claim is only worth something if both sides see the same registry. | +| `ChunkLightServiceIntegrationTest` | The service reads real chunks and writes through `Light#set`. | +| `ChunkLightServiceConcurrencyTest` | The same, from several threads at once. | + +`LightEngineEquivalenceTest` reaches `BlockLight.buildInternalQueue` and `LightCompute.compute` +through reflection rather than by placing a test inside a Minestom package, so no package of the +server is split across two artifacts and the queue type of the built-in path stays off the test +classpath. diff --git a/docs/research/README.md b/docs/research/README.md new file mode 100644 index 00000000..6dd95988 --- /dev/null +++ b/docs/research/README.md @@ -0,0 +1,31 @@ +# Research notes + +Findings from three multi-agent investigations run while building the experimental Anvil chunk +loader. They are kept because each one answers a question that cost real effort to answer and that +will be asked again: *can we replace this part of Minestom, and is it worth it?* + +Every claim in these documents was verified against the sources of Minestom `2026.06.20-26.1.2` or +by compiling and running probe code against that jar. Where a number is a measurement, the document +says so and names the probe. Where the agents disagreed, both positions are recorded rather than +silently resolved. + +| Document | Question | Verdict | +| --- | --- | --- | +| [exception-hierarchy.md](exception-hierarchy.md) | A dedicated checked/unchecked exception hierarchy for the Anvil package | Feasible, design ready, one open decision | +| [instance-container.md](instance-container.md) | A multithreaded, "1:1 compatible" `InstanceContainer` replacement | **Partial** — compiles and runs, but 1:1 compatibility is not reachable and the performance premise is wrong | +| [light-engine.md](light-engine.md) | A faster, lower-memory light engine | **Feasible and measurably faster**, but worth nothing for pre-lit worlds | + +## The recurring lesson + +Three of these investigations started from "replace component X of Minestom". The answers differed +sharply, and the difference was never obvious in advance: + +- **Palette** (investigated earlier, no document): impossible. `sealed interface Palette permits + PaletteImpl` is a hard compiler error, and `Section` is a record holding that exact type. +- **`InstanceContainer`**: possible but pointless in the intended form — the parallelism the request + targeted lives somewhere else entirely. +- **`Light`**: possible, and a prototype was 3.1×–5.8× faster with bit-identical output — but the + code path does not execute at all for the workload Aves actually has. + +The pattern: *sealed-ness decides whether it is possible, and the profile decides whether it is +worth it.* Both have to be checked before designing anything, and neither can be guessed. diff --git a/docs/research/exception-hierarchy.md b/docs/research/exception-hierarchy.md new file mode 100644 index 00000000..3f834ab6 --- /dev/null +++ b/docs/research/exception-hierarchy.md @@ -0,0 +1,118 @@ +# Research: exception hierarchy for the Anvil package + +**Question.** Replace the generic JDK exceptions in `net.theevilreaper.aves.instance.anvil` with a +dedicated hierarchy that has both checked and unchecked types, so that "the server never loads a +world in a strange state". + +**Status.** Design complete, not implemented. One decision is open and is documented below. + +Three agents worked on this: hierarchy semantics, a complete catalogue of every throw site, and the +world-consistency guarantees. They agreed on almost everything and disagreed on one point that +changes the whole migration. + +## What Java allows here + +Verified by compiling probe code with javac 25.0.3. + +**A common root over checked *and* unchecked is impossible as a class.** Java has exactly two roots, +`Exception` and `RuntimeException`, and a class cannot be both. The only bracket is an interface — +but it cannot be caught: + +``` +catch (AnvilFault f) + -> error: incompatible types: AnvilFault cannot be converted to Throwable +``` + +An interface therefore only helps *after* a broad catch, for `instanceof` or a pattern switch. It is +not a way to catch both families in one clause. This is worth stating plainly because the opposite +is a natural assumption. + +**Sealed exceptions compile, but give no exhaustiveness in `catch`.** A `try` block that catches both +permitted subtypes still does not satisfy the compiler; a catch of the sealed supertype remains +necessary. Java has no exhaustiveness analysis for catch clauses. Sealing is still worth it for two +other reasons: it prevents a downstream project from breaking a pattern switch, and the repository +already uses `sealed` in eight places (`MapEntry`, `ClickHolder`, `IItem`, `InventoryLayout`, …). + +**All sealed members must live in one package.** Aves has no `module-info.java`, so it runs in the +unnamed module, where a sealed class may only permit subtypes from its own package. Verified: + +``` +error: class Root in unnamed module cannot extend a sealed class in a different package +``` + +Consequence: either every exception type moves into `…anvil.exception`, or all stay in `…anvil`. +A mix breaks compilation. Moving `AnvilChunkException` is safe — it is `@since 1.16.0` and the +released version is 1.15.2, so nothing depends on it yet. + +**`java.nio.file.Path` must not be stored in an exception field.** It is not serializable +(`NotSerializableException: sun.nio.fs.UnixPath`, verified), which would make the exception +unserializable. Store the region path as a `String`. + +**No `serialVersionUID`.** The repository has zero occurrences, the build sets no `-Xlint`, and these +exceptions never leave the process. + +## The open decision: does the checked root extend `IOException`? + +The two agents that looked at this reached opposite conclusions, and both arguments are sound. + +**For extending `IOException`** — it is a migration question. Roughly 40 `throws` clauses, both +multi-catches in `AvesAnvilLoader`, and 14 `assertThrows(IOException.class)` assertions in the tests +keep working untouched. The change becomes additive. + +**Against extending `IOException`** — it is a correctness question. Every existing +`catch (IOException)` would silently keep catching the new types, including the swallowing block in +`saveChunk`. The migration would then be compile-time only and change nothing at runtime. The cost +(8 signatures in `main`) is exactly what the compiler is for. + +This is a genuine trade-off between migration cost and enforcement, not a case where one side is +wrong. It needs a decision before implementation. + +## Proposed types + +The variant below is the one both agents converged on apart from the inheritance question. Six types +for thirteen classes; both agents explicitly warned against more, since a type nobody catches is +Javadoc maintenance without a decision point. + +| Type | Kind | Purpose | +| --- | --- | --- | +| `AnvilFault` | sealed interface | Common contract, carries the `ChunkLocation`. Not catchable — for pattern switches after a broad catch. | +| `AnvilFormatException` | checked, abstract sealed | Root for everything the file itself got wrong. | +| `RegionFormatException` | checked, final | Broken `.mca` structure: header too short, implausible length field, overlapping sectors, unsupported compression scheme. | +| `ChunkDataException` | checked, final | Broken chunk NBT: missing key, wrong type, empty palette, index outside the palette. | +| `AnvilChunkException` | unchecked, non-sealed | The boundary type. Exists already; gains `implements AnvilFault`. | +| `ChunkLocation` | record | `(chunkX, chunkZ, region, dimension)` — the single definition of the log context. | + +`AnvilChunkException` must be declared `non-sealed` once it implements the sealed interface — a +compiler requirement, and correct in substance since downstream may subclass it. + +## Rules the implementation has to follow + +1. **Exactly one translation point** from checked to unchecked: the catch in `AvesAnvilLoader.loadChunk`, + and its counterpart in `saveChunk`. No other class may wrap checked into unchecked, or the origin + is lost and the double report to the `ExceptionManager` returns. +2. **Programmer errors keep the JDK types.** `BitPacker`, `SectorAllocator` and `PaletteData.singleValue` + keep throwing `IllegalArgumentException` / `IllegalStateException`. Nobody catches them, they lead + into no recovery path, and most are provably unreachable from the disk path. +3. **One exception to rule 2:** `SectorAllocator.reserve` rejecting overlapping sectors *is* reachable + from the disk path via `RegionFile.readHeader`. It is corruption detection, not a programmer error, + and is translated at the `RegionFile` boundary — without changing the allocator, which would cost + it its file independence. +4. **Real IO stays `java.io.IOException`.** There is no explicit throw site for it in the package; + `FileChannel`, `Files` and `channel.force` produce it implicitly. Wrapping adds no knowledge. +5. **No throw without a location.** Every message from the format branch names at least the region + path and the chunk coordinate. `RegionFile` already does this; `NbtReads` and `PaletteData` name + only the key or the number and must have the location passed in — otherwise the tick log reads + `palette must hold at least one entry` with no hint which file it came from. +6. **The context format is defined once**, in `ChunkLocation.toString()`. Exception messages must not + repeat the coordinates as text, or today's double formatting persists. +7. **Never pass coordinates into `NbtReads`, `PaletteData`, `SectionCodec`, `BitPacker`.** These are + deliberately format-only and testable without a running server. The location is attached at the + loader boundary. + +## Style + +New package `…anvil.exception` with a `package-info.java` in the exact repository form +(`@NotNullByDefault` + package + import, four lines). Class Javadoc starting with `The {@link X} is …` +explaining the *why*, plus `@author` / `@version` / `@since`. Constructors `(String)` and +`(String, Throwable)`; the `(Throwable)`-only constructor is deliberately omitted for the format +exceptions, because a format violation without a description is useless. diff --git a/docs/research/instance-container.md b/docs/research/instance-container.md new file mode 100644 index 00000000..b7630dbf --- /dev/null +++ b/docs/research/instance-container.md @@ -0,0 +1,102 @@ +# Research: a multithreaded `InstanceContainer` replacement + +**Question.** Build an `InstanceContainer` that is "1:1 compatible with Minestom" but faster and more +maintainable than the original, developed with DRY, SOLID and TDD, using current Java 25 features. + +**Verdict: partial.** It compiles, it registers, it runs — proven end to end. But *1:1 compatible* is +not reachable, and the performance premise of the request is wrong. Nothing has been implemented. + +## What is possible + +Unlike `Palette`, there is no sealing here. Verified with `javap -v` and reflection against the +actual jar: + +| Type | Modifiers | Sealed? | +| --- | --- | --- | +| `Palette` | `public interface` | **yes** — `PermittedSubclasses: PaletteImpl` | +| `Instance` | `public abstract class` | no, `permitted=null` | +| `InstanceContainer` | `public class` | no, not final, not abstract | + +Both `class AvesInstanceA extends InstanceContainer` and `class AvesInstanceB extends Instance` +compile. `InstanceManager.registerInstance(Instance)` is public, and the `Instance` Javadoc +explicitly invites custom implementations. + +An agent wrote a complete `ForeignInstance extends Instance` in a foreign package — all 19 abstract +methods — compiled it and ran it against a real `MinecraftServer.init()`: + +``` +[1] constructed custom Instance: avesprobe.ForeignInstance +[2] registerInstance() OK, isRegistered=true +[4] loadChunk(0,0) -> DynamicChunk +[5] getBlock(1,64,1) = minecraft:diamond_block +[6] tick() OK +``` + +The scale is manageable: `InstanceContainer` is 776 lines, 58 methods, 16 fields. `Instance` itself +contributes 103 methods that are inherited for free (world border, weather, clocks, entity tracker, +scheduler, event node, snapshots). + +## Why "1:1 compatible" is not reachable + +Seven `instanceof` / cast sites in the whole Minestom codebase couple behaviour to the concrete +`InstanceContainer` type. Four of them silently take a different path for a foreign implementation — +no exception, no warning: + +| Site | Effect on a foreign implementation | +| --- | --- | +| `InstanceManager.unregisterInstance` | Chunks are **not** unloaded and partitions not deleted. Reproduced: `before unregister: chunks=1` → `after unregister: chunks STILL loaded = 1`. A real leak. | +| `SharedInstance` | Field and constructor are typed on `InstanceContainer`. Reproduced: `createSharedInstance(customInstance)` → `ClassCastException`. `areLinked` always returns false. | +| `Chunk` constructor | Computes its shared-viewer list via `instance instanceof InstanceContainer`. A foreign instance gives every chunk a permanently empty list, so `SharedInstance` players never see chunk updates. | +| `ChunkBatch` / `AbsoluteBlockBatch` | `refreshLastBlockChangeTime()` is only called on `InstanceContainer`, so batch/copy semantics drift. | + +Silent divergence is worse than a compile error, because it surfaces as a bug much later. + +There is a second obstacle: **the chunk lifecycle hooks are `protected` and unreachable from a +foreign package.** Deriving from `InstanceContainer` and overriding the hot paths does **not** work. +An escape hatch exists — a custom `Chunk` subclass re-exposing the protected hooks — and it compiles, +but it means the replacement drags a chunk implementation along. + +## Why the performance premise is wrong + +**The chunk and entity tick parallelism does not live in `InstanceContainer`.** It lives in the global +`ThreadDispatcher` of `ServerProcessImpl`, which defaults to a single thread. A replacement container +changes nothing about it. + +What the analysis *did* find is a real and severe concurrency problem — but in locking, not in +threading: + +- **The instance monitor serialises all block writes.** `setBlock` scales negatively under contention. +- **`LightingChunk` takes the same instance monitor**, so a relight freezes the entire instance + (measured factor ≈400,000× in the pathological case). +- **`setBlock().join()` under the instance monitor**: one thread can stall the whole instance + (≈1,600,000×). +- **4 of 9 mutable fields are unsynchronised** (verified with `javap`). +- **`currentlyChangingBlocks` is a `HashMap` guarded by two different locks**; `changingBlockLock` + protects nothing. +- **The chunk locks are `assert`s** — without `-ea` they are inert in production, and Minestom + violates them itself. +- **`chunks` as `Long2ObjectSyncMap`** is the wrong structure for chunk streaming (up to 19.8× slower + in the measured access pattern). + +One earlier suspicion was **refuted**: `retrieveChunk` does not leak `loadingChunks` entries. It does +leave a permanent zombie chunk when `unloadChunk` races a running `loadChunk`, and +`loadChunk().join()` can then hand back a dead chunk. + +## Recommendation from the agents + +Clarify the goal before building anything. *Cleaner and more maintainable* is very achievable — the +original concentrates nine responsibilities in one class. *Faster through multithreading* is not +achievable this way, because the parallelism sits in the dispatcher. + +If it is built, then as `AvesInstance extends Instance` in `net.theevilreaper.aves.instance`, **not** +as an `InstanceContainer` subclass, and with: + +- an own unload path, because `InstanceManager.unregisterInstance` does not clean up foreign types — + this is the one defect that must not be inherited; +- the protected-chunk barrier solved through an own `Chunk` subclass, not through reflection; +- all four `instanceof` break points captured as explicit versioned compatibility tests, so a + Minestom upgrade that adds a fifth is caught by CI; +- the generator path *not* reimplemented in a first version. + +Worthwhile sub-goals in order: a `setBlock` pipeline without the global `synchronized`, consistent +locking for `currentlyChangingBlocks`, and a chunk map suited to streaming. diff --git a/docs/research/light-engine.md b/docs/research/light-engine.md new file mode 100644 index 00000000..0730ab20 --- /dev/null +++ b/docs/research/light-engine.md @@ -0,0 +1,106 @@ +# Research: a faster, lower-memory light engine + +**Question.** Implement light calculation ourselves — faster and using less memory than Minestom's — +with TDD, SOLID, DRY and current Java 25 features. + +**Verdict: feasible, and a prototype was measurably faster with bit-identical output. But for +pre-lit worlds the code path does not execute at all.** Nothing has been implemented in the project. + +## Feasibility: proven, not assumed + +`Light` is **not** sealed (`Light.java:13`, `public interface Light`), unlike `Palette`. All nine +methods are implementable. `Section` is a `public record` with a public canonical constructor, so +`new Section(blockPalette, biomePalette, myLight, myLight)` compiles and runs. + +An agent proved this end to end: a custom `Light` writing the marker byte `0xCD` reached the +`LightData` record that goes to the client: + +``` +chunk class = e2e.E2E$AvesChunk +section skyLight class = e2e.E2E$MarkerLight +LightData skyLight entries=2 blockLight entries=24 +first blockLight[0]=0xcd len=2048 +calculateInternal calls=48 calculateExternal calls=0 +``` + +Coupling is minimal: across all 1454 Minestom sources there are exactly **two** `instanceof BlockLight` +/ `instanceof SkyLight` sites, both inside the stock implementations themselves, and **zero** explicit +casts. `LightingChunk` and `Section` work purely against the interface. If a custom chunk supplier is +the only one in use, those two lines never execute. + +Injection path: `Section` record constructor + the `protected LightingChunk(Instance, int, int, +List
)` constructor + `InstanceContainer.setChunkSupplier`. + +## The finding that decides it + +> **When loading pre-lit Anvil worlds, the light engine does not run at all.** + +`AnvilLoader` reads `SkyLight`/`BlockLight` straight from the NBT and calls `section.skyLight().set(...)`. +Our loader does exactly the same. Measured: `after invalidate() requiresUpdate=true` → +`after set(2048) requiresUpdate=false`. Since `LightingChunk.createLightData` only relights when +`requiresUpdate()` is true, a pre-lit chunk is never recomputed. + +For loading pre-built maps from region files the light engine is therefore **0 %** of the load path, +next to the measured NBT parse 62 %, inflate 28 %, palette 4.5 %. A faster engine speeds that up by +exactly nothing. + +Real cost exists in two other workloads, both measured: + +| Workload | Cost | +| --- | --- | +| Relight of a generated chunk (no stored light) | 2.2 ms/chunk, ≈92 µs/section | +| Runtime `setBlock` (placing a torch) | 0.586 ms per placement | + +## What a custom engine would gain + +A prototype was benchmarked against the real, package-private `LightCompute.compute`: + +| Scenario | Minestom | Prototype | Factor | Divergence | +| --- | --- | --- | --- | --- | +| Realistic block mix (air/stone/leaves/water/glass/dirt) | 576.1 µs | 99.3 µs | **5.8×** | 0 of 4096 cells | +| Adversarial (slabs, stairs, snow, farmland, dirt path, glowstone) | 659.1 µs | 212.9 µs | **3.1×** | 0 of 4096 cells | + +The lever is not the algorithm but the occlusion check. Minestom re-reads the source block in each of +the six directions and resolves palette → `Block.fromStateId` → `registry().occlusionShape()` → +`isOccluded` live. Measured over 3,000,000 iterations: **live 10.95 ns/op versus 0.91 ns/op for a +precomputed bitset — factor 12**, with a 512-byte table per section and zero divergence. + +Minestom's `compute` is a plain FIFO BFS over packed shorts (`[4bit level][4bit y][4bit z][4bit x]`), +using a primitive `ShortArrayFIFOQueue` — no boxing, already clean. Measured allocation is 5.5 KB to +33.7 KB per section, partly because the queue starts at capacity 4 and doubles about 11 times. + +**A correction to the original assumption:** Starlight does *not* use a bucket queue. Its +`TECHNICAL_DETAILS.md` states the queue is a plain FIFO. Its actual principles are a per-entry mask of +which neighbours still need checking ("Vanilla checks ALL 6 neighbours, Starlight checks JUST ONE"), +a per-entry flag whether a shape check is needed at all, and a bitset of guaranteed-opacity-0 blocks +for skylight seeding. + +## Traps + +- **13.70 % of all block types have directional occlusion.** Of 1168 types: 364 uniformly opaque, + 644 uniformly transparent, **160 directional** — slabs, stairs, snow, farmland, dirt paths, + lecterns, daylight detectors, stonecutters. A first draft using a single top-face bitset produced + 472 of 4096 wrong cells. A naive one-bitset design is simply incorrect. +- **`Section.clone()` and `LightingChunk.copy()` silently fall back to Minestom's light.** + `Section.java:27-28` calls `Light.sky()` / `Light.block()` outright. Verified: + `clone() skyLight impl = net.minestom.server.instance.light.SkyLight`. **This affects the existing + Anvil loader**, whose save snapshot uses `section.clone()`. `copy()` would have to be overridden or + the system degrades back unnoticed. +- `calculateInternal` / `calculateExternal` are `@ApiStatus.Internal`, so the adapter surface can + change between Minestom versions. Keep the core engine Minestom-free and the adapter thin — risk + management, not purity. +- Minestom's `content` / `contentPropagation` fields are non-volatile and published across threads. + Do not try to fix Minestom's concurrency model along the way. +- Dead code in the hot path: a `Set sections` is filled and never read. + +## Recommendation from the agents + +Build it only if the goal is **block-placement latency** (0.586 ms per torch) or relighting generated +worlds — not chunk load throughput, where it contributes nothing. Keep the core engine free of +Minestom types, and write the parity test against `Light.block()` / `Light.sky()` **first**, before a +single line of engine code; it is demonstrably feasible and is what makes the 3–6× credible rather +than merely claimed. Realistic effort: several days for engine, adapter and test suite, plus ongoing +maintenance against an `@Internal` API. + +Independently of the decision: the analysis suggests adding the light array length check to the Anvil +loader that Minestom performs. diff --git a/settings.gradle.kts b/settings.gradle.kts index 5eddf1f3..13b09af3 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -23,16 +23,36 @@ dependencyResolutionManagement { create("libs") { version("bom", "1.7.2") version("slf4j", "2.0.18") + version("annotations", "26.1.0") + // The mycelium bom does not manage jmh, so both the harness and the gradle plugin + // which owns the jmh source set need an explicit version here. + version("jmh", "1.37") + version("jmhPlugin", "0.7.3") + // The main source set receives adventure through minestom, which is a compileOnly + // dependency and therefore never reaches a runtime classpath. The benchmarks run their + // code for real and need adventure at runtime, so they import the adventure platform + // directly. Keep this in sync with the version minestom resolves to. + version("adventureBom", "5.1.1") + + plugin("jmh", "me.champeau.jmh").versionRef("jmhPlugin") library("mycelium.bom", "net.onelitefeather", "mycelium-bom").versionRef("bom") library("slf4j.api", "org.slf4j", "slf4j-api").versionRef("slf4j") + library("annotations", "org.jetbrains", "annotations").versionRef("annotations") + // Only the comparison benchmark needs this. Minestom declares it at runtime scope, so it + // is not on a compile classpath, but that benchmark calls a Minestom method taking one. + library("fastutil", "it.unimi.dsi", "fastutil").version("8.5.18") library("minestom","net.minestom", "minestom").withoutVersion() library("adventure", "net.kyori", "adventure-text-minimessage").withoutVersion() + library("adventure.nbt", "net.kyori", "adventure-nbt").withoutVersion() + library("adventure.bom", "net.kyori", "adventure-bom").versionRef("adventureBom") library("cyano", "net.onelitefeather", "cyano").withoutVersion() library("junit-jupiter", "org.junit.jupiter", "junit-jupiter").withoutVersion() library("junit-jupiter-engine", "org.junit.jupiter", "junit-jupiter-engine").withoutVersion() library("junit.platform.launcher", "org.junit.platform", "junit-platform-launcher").withoutVersion() + + library("jmh.core", "org.openjdk.jmh", "jmh-core").versionRef("jmh") } } } diff --git a/src/jmh/java/net/minestom/server/instance/anvil/RegionFileComparisonBenchmark.java b/src/jmh/java/net/minestom/server/instance/anvil/RegionFileComparisonBenchmark.java new file mode 100644 index 00000000..cccb37ae --- /dev/null +++ b/src/jmh/java/net/minestom/server/instance/anvil/RegionFileComparisonBenchmark.java @@ -0,0 +1,302 @@ +package net.minestom.server.instance.anvil; + +import net.kyori.adventure.nbt.BinaryTagIO; +import net.kyori.adventure.nbt.CompoundBinaryTag; +import net.theevilreaper.aves.benchmark.support.BenchmarkConstants; +import net.theevilreaper.aves.benchmark.support.ChunkColumn; +import net.theevilreaper.aves.benchmark.support.ChunkPayloads; +import net.theevilreaper.aves.benchmark.support.FakePaletteEntryResolver; +import net.theevilreaper.aves.instance.anvil.ChunkCompression; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +/** + * The {@link RegionFileComparisonBenchmark} class measures the region file of Aves against the one + * Minestom ships with, on the same bytes and through the same compression code. + *

+ * The class lives in the Minestom anvil package because {@code net.minestom.server.instance.anvil.RegionFile} + * is package-private. This is the only way to measure the original rather than a reimplementation + * of it, the same reason the light engine comparison lives in the Minestom light package. + *

+ *

+ * The region file is the layer at which the two loaders can be compared without a running server. + * Minestom's {@code AnvilLoader} cannot be touched from a bare benchmark fork at all: its static + * fields read the biome registry and the block state count, so the class initialiser fails before + * any measurement starts. Its region file reads no registry and is therefore measurable directly. + *

+ *

+ * Both sides are measured over the same total work, from a stored chunk to a parsed compound and + * back. The difference between them is where that work happens relative to the lock of the file: + *

+ *
    + *
  • Minestom reads the bytes, inflates them and parses the NBT with the file lock held, so two + * readers of the same region file cannot overlap at all.
    • + *
  • Aves reads the bytes through positional channel operations without any lock and validates + * the read against a version counter afterwards, so only the inflate and the parse of a + * reader overlap with those of another one.
    • + *
  • Minestom rewrites the whole {@code 8192} byte header on every write, Aves rewrites the + * eight bytes of the affected entry.
    • + *
+ *

+ * Fairness of the compression is enforced rather than assumed. Both sides run the identical + * Adventure writer at its default level, and the payload both files hold is byte for byte the same, + * so nothing of what is measured here is a compression level in disguise. The loader of Aves ships + * a lower level by default, which is a property of the loader and not of the region file, and is + * therefore measured by {@code ChunkSaveComparisonBenchmark} instead. + *

+ *

+ * The thread count is the axis that carries this benchmark. Run it over a series with the + * {@code -t} option of the harness, because a single thread cannot show a difference that only + * exists between threads. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(value = 1, jvmArgsAppend = {"-Xms1g", "-Xmx1g"}) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +public class RegionFileComparisonBenchmark { + + /** + * The amount of chunks the measured region file holds. + *

+ * Every thread works on a chunk of its own so the threads never collide on a single entry. + * What they still share is the region file, which is exactly the resource the two + * implementations guard differently. + *

+ */ + private static final int CHUNK_COUNT = 32; + + /** + * The width of the chunk grid inside the region file. + */ + private static final int GRID_WIDTH = 8; + + private static final BinaryTagIO.Reader TAG_READER = BinaryTagIO.unlimitedReader(); + private static final BinaryTagIO.Writer TAG_WRITER = BinaryTagIO.writer(); + + /** + * Hands every benchmark thread a chunk of its own. + */ + private static final AtomicInteger SLOTS = new AtomicInteger(); + + /** + * The amount of distinct block states a single section of the measured chunk holds. + *

+ * The value decides how large the stored payload is and therefore how much of a read is the + * inflate and the parse rather than the transfer of the bytes. + *

+ */ + @Param({"8", "200"}) + public int distinctStates; + + private Path directory; + private RegionFile minestomRegion; + private net.theevilreaper.aves.instance.anvil.RegionFile avesRegion; + private CompoundBinaryTag chunkData; + + /** + * Creates a new benchmark instance. + */ + public RegionFileComparisonBenchmark() { + } + + /** + * Builds one chunk compound and stores it in both region files, with byte identical payloads. + *

+ * The payload is produced once with the Adventure writer both implementations use and handed to + * both files, so a later read of either side decodes the very same bytes. + *

+ * + * @throws IOException if the region files cannot be prepared + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + this.directory = Files.createTempDirectory("aves-region-comparison"); + + ChunkColumn column = ChunkColumn.of(BenchmarkConstants.OVERWORLD_SECTIONS, this.distinctStates); + FakePaletteEntryResolver resolver = new FakePaletteEntryResolver(); + this.chunkData = ChunkPayloads.encode(column, resolver, resolver); + + ByteArrayOutputStream compressed = new ByteArrayOutputStream(64 * 1024); + TAG_WRITER.writeNamed(Map.entry("", this.chunkData), compressed, BinaryTagIO.Compression.ZLIB); + byte[] payload = compressed.toByteArray(); + + this.minestomRegion = new RegionFile(this.directory.resolve("minestom.mca")); + this.avesRegion = net.theevilreaper.aves.instance.anvil.RegionFile.open(this.directory.resolve("aves.mca")); + + for (int slot = 0; slot < CHUNK_COUNT; slot++) { + int chunkX = chunkX(slot); + int chunkZ = chunkZ(slot); + this.minestomRegion.writeChunkData(chunkX, chunkZ, this.chunkData); + this.avesRegion.writeRaw(chunkX, chunkZ, ChunkCompression.ZLIB, payload); + } + } + + /** + * Closes both region files and removes the temporary directory. + * + * @throws IOException if the directory cannot be removed + */ + @TearDown(Level.Trial) + public void tearDown() throws IOException { + this.minestomRegion.close(); + this.avesRegion.close(); + + try (Stream entries = Files.walk(this.directory)) { + entries.sorted(Comparator.reverseOrder()).forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException ignored) { + // A leftover file in the temporary directory does not invalidate a measurement. + } + }); + } + } + + /** + * Reads a stored chunk through the region file of Minestom. + * The lock of the file is held for the transfer, the inflate and the parse. + * + * @param slot the chunk this thread works on + * @return the parsed chunk data + * @throws IOException if the chunk cannot be read + */ + @Benchmark + public CompoundBinaryTag minestomRead(ThreadSlot slot) throws IOException { + return this.minestomRegion.readChunkData(slot.chunkX, slot.chunkZ); + } + + /** + * Reads the same stored chunk through the region file of Aves. + * Only the transfer of the bytes is guarded, the inflate and the parse run without a lock. + * + * @param slot the chunk this thread works on + * @return the parsed chunk data + * @throws IOException if the chunk cannot be read + */ + @Benchmark + public CompoundBinaryTag avesRead(ThreadSlot slot) throws IOException { + net.theevilreaper.aves.instance.anvil.RegionFile.RawChunk raw = + this.avesRegion.readRaw(slot.chunkX, slot.chunkZ); + return TAG_READER.read(new ByteArrayInputStream(raw.decompress()), BinaryTagIO.Compression.NONE); + } + + /** + * Writes a chunk through the region file of Minestom, which rewrites the whole header. + * + * @param slot the chunk this thread works on + * @throws IOException if the chunk cannot be written + */ + @Benchmark + public void minestomWrite(ThreadSlot slot) throws IOException { + this.minestomRegion.writeChunkData(slot.chunkX, slot.chunkZ, this.chunkData); + } + + /** + * Writes the same chunk through the region file of Aves, which rewrites the affected entry. + *

+ * The serialisation and the compression happen inside the measured method on purpose. They do + * so on the Minestom side as well, so both sides carry the identical amount of work and only + * the part behind the lock differs. + *

+ * + * @param slot the chunk this thread works on + * @throws IOException if the chunk cannot be written + */ + @Benchmark + public void avesWrite(ThreadSlot slot) throws IOException { + ByteArrayOutputStream target = new ByteArrayOutputStream(64 * 1024); + TAG_WRITER.writeNamed(Map.entry("", this.chunkData), target, BinaryTagIO.Compression.ZLIB); + this.avesRegion.writeRaw(slot.chunkX, slot.chunkZ, ChunkCompression.ZLIB, target.toByteArray()); + } + + /** + * Returns the x coordinate of the chunk which belongs to the given slot. + * + * @param slot the slot to resolve + * @return the absolute chunk x coordinate + */ + private static int chunkX(int slot) { + return slot % GRID_WIDTH; + } + + /** + * Returns the z coordinate of the chunk which belongs to the given slot. + * + * @param slot the slot to resolve + * @return the absolute chunk z coordinate + */ + private static int chunkZ(int slot) { + return slot / GRID_WIDTH; + } + + /** + * The {@link ThreadSlot} class assigns a chunk of the shared region file to a benchmark thread. + *

+ * Without it every thread would hammer the same entry, which measures the contention on one + * chunk rather than the contention on the file. Real parallel loading reads different chunks of + * the same region, which is what a slot per thread reproduces. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ + @State(Scope.Thread) + public static class ThreadSlot { + + /** + * The absolute chunk x coordinate this thread works on. + */ + public int chunkX; + + /** + * The absolute chunk z coordinate this thread works on. + */ + public int chunkZ; + + /** + * Creates a new thread slot. + */ + public ThreadSlot() { + } + + /** + * Picks the chunk of this thread. + */ + @Setup(Level.Trial) + public void setUp() { + int slot = SLOTS.getAndIncrement() % CHUNK_COUNT; + this.chunkX = chunkX(slot); + this.chunkZ = chunkZ(slot); + } + } +} diff --git a/src/jmh/java/net/minestom/server/instance/light/LightEngineComparisonBenchmark.java b/src/jmh/java/net/minestom/server/instance/light/LightEngineComparisonBenchmark.java new file mode 100644 index 00000000..ebab3448 --- /dev/null +++ b/src/jmh/java/net/minestom/server/instance/light/LightEngineComparisonBenchmark.java @@ -0,0 +1,320 @@ +package net.minestom.server.instance.light; + +import it.unimi.dsi.fastutil.shorts.ShortArrayFIFOQueue; +import net.minestom.server.MinecraftServer; +import net.minestom.server.instance.block.Block; +import net.minestom.server.instance.palette.Palette; +import net.theevilreaper.aves.instance.light.LightNibbles; +import net.theevilreaper.aves.instance.light.LightPropagator; +import net.theevilreaper.aves.instance.light.MinestomBlockLightSource; +import net.theevilreaper.aves.instance.light.SectionOpacity; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.Arrays; +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * The {@link LightEngineComparisonBenchmark} class measures the light engine of Aves against the one + * Minestom ships with, on the same section and to the same result. + *

+ * The class lives in the Minestom light package because the two methods that form the built-in path, + * {@code BlockLight.buildInternalQueue} and {@code LightCompute.compute}, are package-private. This + * is the only way to measure the original rather than a reimplementation of it. + *

+ *

+ * Both sides are measured over their full path, from a block palette to a finished light array of + * {@code 2048} bytes. Neither side gets to skip its preparation: the built-in path builds its seed + * queue, and the Aves path builds its opacity table through the real block registry rather than a + * stand-in. Measuring only the searches would flatter whichever side does more of its work up front. + *

+ * + *

Why the brightness of the sources is a parameter

+ *

+ * The propagation of Aves is a breadth-first search that assumes the queued positions are ordered by + * their level, which only holds while every source starts at the same one. {@code LightPropagator} + * says so where it grows its queue, and {@code LightEngineConcurrencyTest} says so where it pins its + * fixtures to a single level. Mixing levels makes the search revisit positions, and the amount of + * queued entries stops being bounded by the amount of positions. + *

+ *

+ * That distinction decides whether a bucket queue is worth having. A bucket queue pops the brightest + * position first and therefore touches every position once, at the price of a bucket per level. It + * is the wrong trade when every source is equally bright, because the ordering it buys is already + * there for free, and the right one as soon as the levels differ. A benchmark that only ever places + * one kind of source would report such a change as a regression while real worlds hold the case in + * which it wins, so the mixture is a parameter here rather than a constant. + *

+ * + *

The emission levels the mixture uses

+ *

+ * {@link EmissionMix#MIXED} places the blocks a real world actually lights its interiors with, at + * the levels the block registry gives them: glowstone {@code 15}, lantern {@code 15}, torch + * {@code 14}, redstone torch {@code 7} and magma block {@code 3}. Two properties of that set are + * deliberate. It spans almost the whole range, from {@code 3} to {@code 15}, so a search that + * expects one level meets the worst spread a builder can produce without exotic blocks; and it holds + * two pairs that sit one level apart or on the same level, {@code 15}/{@code 15} and + * {@code 15}/{@code 14}, because a mixture of neighbouring levels costs a search far less than one + * of distant levels and a set of only distant levels would overstate the effect. Torch, lantern and + * redstone torch occlude nothing while glowstone and magma block occlude every face, so the set also + * carries both occlusion shapes an emitter can have. + *

+ * + *

The two engines have to agree

+ *

+ * Every trial verifies that both paths produce the same {@code 2048} bytes before a single + * measurement is taken, and fails the trial if they do not. A faster number must never come from + * computing something else, and a mixture of levels is exactly the input on which an order dependent + * search would start to drift away from the reference. + *

+ * + *

Running it

+ *

+ * The full cross product is {@code 3 x 2 x 2} scenarios per method. One of them measures nothing: + * with a single source the mixture degenerates, because the first block of the set is glowstone and + * a lone source is placed at the same position with the same level as under + * {@link EmissionMix#UNIFORM}. The recommended run therefore leaves it out: + *

+ * 
{@code
+ * java -jar build/libs/aves-*-jmh.jar "LightEngineComparisonBenchmark.(aves|minestom)" \
+ *     -p emissionMix=UNIFORM -f 1 -wi 3 -i 5
+ * java -jar build/libs/aves-*-jmh.jar "LightEngineComparisonBenchmark.(aves|minestom)" \
+ *     -p emissionMix=MIXED -p lightSources=8,64 -f 1 -wi 3 -i 5
+ * }
+ *

+ * The first line reproduces the six documented scenarios unchanged, the second adds the four new + * ones, which is ten scenarios per method instead of the twelve a plain run would take. + *

+ * + * @author TheMeinerLP + * @version 1.1.0 + * @since 1.16.0 + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(value = 1, jvmArgsAppend = {"-Xms512m", "-Xmx512m"}) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +public class LightEngineComparisonBenchmark { + + private static final int DIMENSION = 16; + private static final int BLOCK_COUNT = DIMENSION * DIMENSION * DIMENSION; + private static final long SEED = 20260731L; + + /** + * The brightness the light emitting blocks of the measured section carry. + *

+ * The two constants differ in nothing but the blocks that are placed. The positions of the + * sources are drawn from the same seeded sequence either way, so a pair of scenarios that shares + * its source count and its occlusion share the very same section apart from the levels. + *

+ */ + public enum EmissionMix { + + /** + * Every source is a glowstone block and emits level {@code 15}. + * This is the form the benchmark had before the mixture became a parameter, down to the + * drawn positions, so the scenarios measured under it stay comparable to the documented + * numbers. + */ + UNIFORM, + + /** + * The sources cycle through glowstone, lantern, torch, redstone torch and magma block, which + * emit {@code 15}, {@code 15}, {@code 14}, {@code 7} and {@code 3}. + * Eight sources already cover every one of those levels. + */ + MIXED + } + + /** + * The amount of light emitting blocks the measured section holds. + */ + @Param({"1", "8", "64"}) + public int lightSources; + + /** + * The share of solid blocks in the measured section, in percent. + */ + @Param({"0", "30"}) + public int occlusionPercent; + + /** + * Whether the sources of the measured section are equally bright or of mixed brightness. + */ + @Param({"UNIFORM", "MIXED"}) + public EmissionMix emissionMix; + + private Palette palette; + private int[] stateIds; + private MinestomBlockLightSource source; + private LightPropagator propagator; + + /** + * Starts the server once so the block registry is available, builds the section both sides are + * measured on and verifies that both sides agree on its light. + * + * @throws IllegalStateException if the two engines calculate different light for the section + */ + @Setup(Level.Trial) + public void setUp() { + if (MinecraftServer.process() == null) { + MinecraftServer.init(); + } + + this.source = new MinestomBlockLightSource(); + this.propagator = new LightPropagator(); + this.stateIds = new int[BLOCK_COUNT]; + + Random random = new Random(SEED); + int air = Block.AIR.stateId(); + int stone = Block.STONE.stateId(); + int[] emitters = emittersOf(this.emissionMix); + + for (int index = 0; index < BLOCK_COUNT; index++) { + this.stateIds[index] = random.nextInt(100) < this.occlusionPercent ? stone : air; + } + // The block is picked by the round the source is placed in rather than by another draw, so + // the drawn positions stay the same for both mixtures and the levels are the only difference + // between them. + for (int placed = 0; placed < this.lightSources; placed++) { + this.stateIds[random.nextInt(BLOCK_COUNT)] = emitters[placed % emitters.length]; + } + + this.palette = Palette.blocks(); + this.palette.setAll((x, y, z) -> this.stateIds[(y << 8) | (z << 4) | x]); + + verifyBothEnginesAgree(); + } + + /** + * Measures the built-in path: building the seed queue and running the search of Minestom. + * + * @return the calculated light array of the section + */ + @Benchmark + public byte[] minestom() { + ShortArrayFIFOQueue queue = BlockLight.buildInternalQueue(this.palette); + return LightCompute.compute(this.palette, queue); + } + + /** + * Measures the Aves path: building the opacity table through the real registry and running the + * search, ending in the same light array layout. + * + * @return the calculated light array of the section + */ + @Benchmark + public byte[] aves() { + int[] states = new int[BLOCK_COUNT]; + this.palette.getAll((x, y, z, value) -> states[(y << 8) | (z << 4) | x] = value); + LightNibbles light = this.propagator.propagate(SectionOpacity.of(states, this.source)); + return light.toDenseArray(); + } + + /** + * Returns the blocks the sources of the given mixture are placed as. + *

+ * Glowstone comes first so that a section with a single source holds the same block under both + * mixtures, which is what makes that scenario a duplicate rather than a second measurement. + *

+ * + * @param mix the mixture to build the blocks for + * @return the state ids the sources cycle through + */ + private static int[] emittersOf(EmissionMix mix) { + if (mix == EmissionMix.UNIFORM) { + return new int[]{Block.GLOWSTONE.stateId()}; + } + return new int[]{ + Block.GLOWSTONE.stateId(), + Block.LANTERN.stateId(), + Block.TORCH.stateId(), + Block.REDSTONE_TORCH.stateId(), + Block.MAGMA_BLOCK.stateId() + }; + } + + /** + * Verifies that both engines calculate the same light for the section that was just built. + *

+ * The check runs once per trial, before any measurement, and stops the trial when the results + * differ. Without it a change to either engine could win time by no longer computing the same + * thing, and the numbers of the two sides would stop describing the same task. + *

+ * + * @throws IllegalStateException if the two engines calculate different light for the section + */ + private void verifyBothEnginesAgree() { + byte[] expected = minestom(); + byte[] actual = aves(); + + if (Arrays.equals(expected, actual)) { + return; + } + throw new IllegalStateException(describeDifference(expected, actual)); + } + + /** + * Describes how far apart the light of the two engines is. + * + * @param expected the light the built-in engine calculated + * @param actual the light the Aves engine calculated + * @return a message naming the amount of differing blocks, the largest difference and the first + * block the two engines disagree on + */ + private String describeDifference(byte[] expected, byte[] actual) { + if (expected.length != actual.length) { + return "The engines returned light arrays of different length: Minestom " + expected.length + + " bytes against Aves " + actual.length + " bytes"; + } + + int differing = 0; + int largest = 0; + int first = -1; + + for (int index = 0; index < BLOCK_COUNT; index++) { + int expectedLevel = levelAt(expected, index); + int actualLevel = levelAt(actual, index); + + if (expectedLevel == actualLevel) { + continue; + } + differing++; + largest = Math.max(largest, Math.abs(expectedLevel - actualLevel)); + + if (first < 0) { + first = index; + } + } + return "The engines disagree on " + differing + " of " + BLOCK_COUNT + " blocks, largest difference " + + largest + " levels, first at x=" + (first & 15) + " y=" + ((first >> 8) & 15) + + " z=" + ((first >> 4) & 15) + " where Minestom holds " + levelAt(expected, first) + + " and Aves holds " + levelAt(actual, first) + " (lightSources=" + this.lightSources + + ", occlusionPercent=" + this.occlusionPercent + ", emissionMix=" + this.emissionMix + ")"; + } + + /** + * Reads the light level of a block out of a light array. + * Both engines store two levels per byte in the same order, so one reader serves both. + * + * @param light the light array to read from + * @param index the index of the block inside the section + * @return the level of the block + */ + private static int levelAt(byte[] light, int index) { + return (light[index >> 1] >> ((index & 1) << 2)) & 0x0F; + } +} diff --git a/src/jmh/java/net/minestom/server/instance/light/LightEngineStageBenchmark.java b/src/jmh/java/net/minestom/server/instance/light/LightEngineStageBenchmark.java new file mode 100644 index 00000000..6db43c0f --- /dev/null +++ b/src/jmh/java/net/minestom/server/instance/light/LightEngineStageBenchmark.java @@ -0,0 +1,203 @@ +package net.minestom.server.instance.light; + +import it.unimi.dsi.fastutil.shorts.ShortArrayFIFOQueue; +import net.minestom.server.MinecraftServer; +import net.minestom.server.instance.block.Block; +import net.minestom.server.instance.palette.Palette; +import net.theevilreaper.aves.instance.light.LightNibbles; +import net.theevilreaper.aves.instance.light.LightPropagator; +import net.theevilreaper.aves.instance.light.MinestomBlockLightSource; +import net.theevilreaper.aves.instance.light.SectionOpacity; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * The {@link LightEngineStageBenchmark} class splits both light engines into the stages they consist + * of and measures every stage on its own. + *

+ * {@link LightEngineComparisonBenchmark} reports one number per engine, which says which one is + * faster but never says why. This class exists to answer the second question: it measures the same + * scenarios on the same section, but one stage at a time, so a difference between the two engines + * can be attributed to a stage instead of being guessed at. + *

+ *

+ * The stages of the Aves path are reading the palette into an array of state ids, building the + * opacity table from those ids, running the breadth-first search and packing the result into the + * dense array the server expects. Every stage receives the finished output of the previous one from + * the setup, so a measured method really only performs the stage it is named after. + *

+ *

+ * The built-in path is split into building the seed queue and everything else. Its search consumes + * the queue it is handed, so the search cannot be measured on a prepared queue without rebuilding + * that queue per invocation, which would put the rebuild back into the measurement. The search is + * therefore the difference between {@link #minestomFull()} and {@link #minestomQueue()}. + *

+ *

+ * The class lives in the Minestom light package for the same reason + * {@link LightEngineComparisonBenchmark} does: {@code BlockLight.buildInternalQueue} and + * {@code LightCompute.compute} are package-private. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(value = 1, jvmArgsAppend = {"-Xms512m", "-Xmx512m"}) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +public class LightEngineStageBenchmark { + + private static final int DIMENSION = 16; + private static final int BLOCK_COUNT = DIMENSION * DIMENSION * DIMENSION; + private static final long SEED = 20260731L; + + /** + * The amount of light emitting blocks the measured section holds. + */ + @Param({"1", "8", "64"}) + public int lightSources; + + /** + * The share of solid blocks in the measured section, in percent. + */ + @Param({"0", "30"}) + public int occlusionPercent; + + private Palette palette; + private int[] stateIds; + private MinestomBlockLightSource source; + private LightPropagator propagator; + private SectionOpacity opacity; + private LightNibbles light; + + /** + * Creates a new benchmark instance. + */ + public LightEngineStageBenchmark() { + } + + /** + * Starts the server once so the block registry is available, builds the section both engines are + * measured on and prepares the input of every single stage. + */ + @Setup(Level.Trial) + public void setUp() { + if (MinecraftServer.process() == null) { + MinecraftServer.init(); + } + + this.source = new MinestomBlockLightSource(); + this.propagator = new LightPropagator(); + this.stateIds = new int[BLOCK_COUNT]; + + Random random = new Random(SEED); + int air = Block.AIR.stateId(); + int stone = Block.STONE.stateId(); + int glowstone = Block.GLOWSTONE.stateId(); + + for (int index = 0; index < BLOCK_COUNT; index++) { + this.stateIds[index] = random.nextInt(100) < this.occlusionPercent ? stone : air; + } + for (int placed = 0; placed < this.lightSources; placed++) { + this.stateIds[random.nextInt(BLOCK_COUNT)] = glowstone; + } + + this.palette = Palette.blocks(); + this.palette.setAll((x, y, z) -> this.stateIds[(y << 8) | (z << 4) | x]); + + this.opacity = SectionOpacity.of(this.stateIds, this.source); + this.light = new LightPropagator().propagate(this.opacity); + } + + /** + * Measures reading the block palette into an array of state ids. + * + * @return the read state ids + */ + @Benchmark + public int[] avesReadStates() { + int[] states = new int[BLOCK_COUNT]; + this.palette.getAll((x, y, z, value) -> states[(y << 8) | (z << 4) | x] = value); + return states; + } + + /** + * Measures building the opacity table from an already read array of state ids. + * + * @return the created table + */ + @Benchmark + public SectionOpacity avesOpacity() { + return SectionOpacity.of(this.stateIds, this.source); + } + + /** + * Measures the breadth-first search on an already built opacity table. + * + * @return the calculated light of the section + */ + @Benchmark + public LightNibbles avesPropagate() { + return this.propagator.propagate(this.opacity); + } + + /** + * Measures packing an already calculated result into the dense array the server expects. + * + * @return the packed light array + */ + @Benchmark + public byte[] avesCollect() { + return this.light.toDenseArray(); + } + + /** + * Measures the whole Aves path so the sum of the stages can be checked against it. + * + * @return the calculated light array of the section + */ + @Benchmark + public byte[] avesFull() { + int[] states = new int[BLOCK_COUNT]; + this.palette.getAll((x, y, z, value) -> states[(y << 8) | (z << 4) | x] = value); + LightNibbles result = this.propagator.propagate(SectionOpacity.of(states, this.source)); + return result.toDenseArray(); + } + + /** + * Measures building the seed queue of the built-in path. + * + * @return the built seed queue + */ + @Benchmark + public ShortArrayFIFOQueue minestomQueue() { + return BlockLight.buildInternalQueue(this.palette); + } + + /** + * Measures the whole built-in path. Its search is the difference to {@link #minestomQueue()}. + * + * @return the calculated light array of the section + */ + @Benchmark + public byte[] minestomFull() { + ShortArrayFIFOQueue queue = BlockLight.buildInternalQueue(this.palette); + return LightCompute.compute(this.palette, queue); + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/ScalingBenchmark.java b/src/jmh/java/net/theevilreaper/aves/benchmark/ScalingBenchmark.java new file mode 100644 index 00000000..cee2f888 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/ScalingBenchmark.java @@ -0,0 +1,143 @@ +package net.theevilreaper.aves.benchmark; + +import net.theevilreaper.aves.benchmark.support.FakeBlockLightSource; +import net.theevilreaper.aves.benchmark.support.SectionStates; +import net.theevilreaper.aves.instance.anvil.BitPacker; +import net.theevilreaper.aves.instance.anvil.PaletteData; +import net.theevilreaper.aves.instance.light.ChunkLightPropagator; +import net.theevilreaper.aves.instance.light.LightNibbles; +import net.theevilreaper.aves.instance.light.SectionOpacity; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * The {@link ScalingBenchmark} class measures how the cost of the hot operations grows with the + * size of their input. + *

+ * The other benchmarks answer "how long does this take". This one answers "what happens when it + * gets bigger", which is the question that decides whether a workload is viable at ten times its + * current size. Every axis therefore uses many closely spaced sizes rather than a few + * representative ones, so the shape of the curve becomes visible and can be extrapolated. + *

+ *

+ * Reading the result: a straight line through the points means the operation is linear in that + * axis and a forecast is a simple multiplication. A curve that bends upwards means it is not, and + * the workload has a size beyond which it stops being affordable. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(value = 1, jvmArgsAppend = {"-Xms512m", "-Xmx512m"}) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 3, time = 1) +public class ScalingBenchmark { + + /** + * The amount of sections a chunk column holds. + *

+ * Twenty four is the height of a modern overworld and the value a normal server runs at. The + * points beyond it describe worlds with a raised height limit, which servers do configure: a + * value of 128 is a column of 2048 blocks, and 256 is 4096. Measuring them instead of + * extrapolating from the vanilla range is the point of this axis, because a forecast that is + * only ever validated inside the common range says nothing about the exotic one. + *

+ */ + @Param({"1", "2", "4", "8", "12", "16", "20", "24", "32", "48", "64", "96", "128", "192", "256"}) + public int sectionCount; + + /** + * The amount of distinct block states a section holds. A palette grows with the variety of a + * build, and the cost of resolving it is the claim this axis verifies. + */ + @Param({"1", "16", "64", "256", "1024"}) + public int distinctStates; + + private List litColumn; + private List openColumn; + private ChunkLightPropagator propagator; + private int[] paletteValues; + + /** + * Builds the inputs of every axis once, so no benchmark measures its own setup. + */ + @Setup + public void setUp() { + FakeBlockLightSource source = new FakeBlockLightSource(); + this.propagator = new ChunkLightPropagator(); + this.litColumn = new ArrayList<>(this.sectionCount); + this.openColumn = new ArrayList<>(this.sectionCount); + + for (int section = 0; section < this.sectionCount; section++) { + this.litColumn.add(SectionOpacity.of(SectionStates.lit(4, 20), source)); + this.openColumn.add(SectionOpacity.of(SectionStates.uniform(LightNibbles.BLOCK_COUNT, FakeBlockLightSource.AIR), source)); + } + + this.paletteValues = new int[LightNibbles.BLOCK_COUNT]; + + for (int index = 0; index < this.paletteValues.length; index++) { + this.paletteValues[index] = index % this.distinctStates; + } + } + + /** + * Measures how the block light search grows with the height of the column. + * + * @return the calculated light of every section + */ + @Benchmark + public List blockLightBySectionCount() { + return this.propagator.propagate(this.litColumn); + } + + /** + * Measures how the sky light search grows with the height of the column. + * An open column seeds nearly every block, so this is the upper bound of a propagation. + * + * @return the calculated sky light of every section + */ + @Benchmark + public List skyLightBySectionCount() { + return this.propagator.propagateSky(this.openColumn); + } + + /** + * Measures how encoding a section grows with the amount of distinct block states it holds. + * + * @return the encoded palette of the section + */ + @Benchmark + public PaletteData paletteByDistinctStates() { + return PaletteData.encode(this.paletteValues, 4); + } + + /** + * Measures how packing grows with the amount of bits a palette entry needs, which is itself a + * function of the amount of distinct states. + * + * @param blackhole the sink which keeps the result from being optimised away + */ + @Benchmark + public void packingByDistinctStates(Blackhole blackhole) { + int bits = BitPacker.bitsPerEntry(this.distinctStates, 4); + blackhole.consume(BitPacker.pack(this.paletteValues, bits)); + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/BitPackerBenchmark.java b/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/BitPackerBenchmark.java new file mode 100644 index 00000000..84a7ac21 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/BitPackerBenchmark.java @@ -0,0 +1,118 @@ +package net.theevilreaper.aves.benchmark.anvil; + +import net.theevilreaper.aves.benchmark.support.BenchmarkConstants; +import net.theevilreaper.aves.instance.anvil.BitPacker; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * The {@link BitPackerBenchmark} class measures the packing and the unpacking of the palette + * indices of a single section. + *

+ * Both methods are pure loops over the 4096 entries of a section and they run once per section on + * every chunk load and on every chunk save. A chunk of a + * full height overworld therefore runs them twenty four times, which makes them the hottest loop of + * the whole codec. + *

+ *

+ * The parameter is the amount of bits a single entry occupies, because that value alone decides how + * many entries share a long and therefore how many iterations the loop performs per long. Four bits + * is the smallest amount the block palette allows, five and eight are what a normal and a busy + * section produce, and fifteen is the direct palette which stores a state id without a palette at + * all. + *

+ *

+ * The time is reported per whole section, not per entry. Dividing by the entry count gives the cost + * of a single entry. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(value = 2, jvmArgsAppend = {"-Xms512m", "-Xmx512m"}) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +public class BitPackerBenchmark { + + /** + * The amount of bits a single palette index occupies. + */ + @Param({"4", "5", "8", "15"}) + public int bitsPerEntry; + + private int[] values; + private long[] packed; + + /** + * Creates a new benchmark instance. + */ + public BitPackerBenchmark() { + } + + /** + * Builds the packed and the unpacked representation of a section. + * Both benchmarks read a prepared input so neither of them measures the generation of it. + */ + @Setup(Level.Trial) + public void setUp() { + Random random = new Random(BenchmarkConstants.SEED); + int bound = this.bitsPerEntry >= Integer.SIZE - 1 ? Integer.MAX_VALUE : 1 << this.bitsPerEntry; + this.values = new int[BenchmarkConstants.BLOCK_ENTRIES]; + + for (int index = 0; index < this.values.length; index++) { + this.values[index] = random.nextInt(bound); + } + this.packed = BitPacker.pack(this.values, this.bitsPerEntry); + } + + /** + * Packs a whole section of palette indices into longs. + * The returned array is handed back to the harness so the loop cannot be removed. + * + * @return the packed representation of the section + */ + @Benchmark + public long[] pack() { + return BitPacker.pack(this.values, this.bitsPerEntry); + } + + /** + * Unpacks a whole section of palette indices out of longs. + * The returned array is handed back to the harness so the loop cannot be removed. + * + * @return the palette indices of the section + */ + @Benchmark + public int[] unpack() { + return BitPacker.unpack(this.packed, BenchmarkConstants.BLOCK_ENTRIES, this.bitsPerEntry); + } + + /** + * Packs and unpacks a section, which is what a load followed by a save performs. + * + * @return the palette indices of the section + */ + @Benchmark + public int[] roundTrip() { + return BitPacker.unpack( + BitPacker.pack(this.values, this.bitsPerEntry), BenchmarkConstants.BLOCK_ENTRIES, this.bitsPerEntry + ); + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/ChunkCompressionBenchmark.java b/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/ChunkCompressionBenchmark.java new file mode 100644 index 00000000..98574d62 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/ChunkCompressionBenchmark.java @@ -0,0 +1,112 @@ +package net.theevilreaper.aves.benchmark.anvil; + +import net.theevilreaper.aves.benchmark.support.BenchmarkConstants; +import net.theevilreaper.aves.benchmark.support.ChunkColumn; +import net.theevilreaper.aves.benchmark.support.ChunkPayloads; +import net.theevilreaper.aves.benchmark.support.FakePaletteEntryResolver; +import net.theevilreaper.aves.instance.anvil.ChunkCompression; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +/** + * The {@link ChunkCompressionBenchmark} class measures the compression and the decompression of a + * complete chunk payload. + *

+ * The payload is not random noise. It is the serialised NBT of a chunk of twenty four sections, + * built through the same codec the save path uses, so its entropy matches what a region file really + * stores. Random bytes would not compress at all and would make zlib look far more expensive than + * it is on real data. + *

+ *

+ * This benchmark carries the weight of the central design claim of the loader. Compression and + * decompression are the most expensive stage of a chunk transfer and the loader performs both of + * them outside of the region lock. The number this benchmark reports is the amount of time that + * would otherwise be spent inside that lock. + *

+ *

+ * The scheme is a parameter because the format allows all three. Vanilla writes zlib, gzip appears + * in older worlds and none appears in worlds which were written by a tool that optimised for speed. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(value = 2, jvmArgsAppend = {"-Xms512m", "-Xmx512m"}) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +public class ChunkCompressionBenchmark { + + /** + * The compression scheme the payload is stored with. + */ + @Param({"ZLIB", "GZIP", "NONE"}) + public ChunkCompression compression; + + /** + * The amount of distinct block states a single section of the chunk holds. + * The value decides how well the payload compresses. + */ + @Param({"8", "200"}) + public int distinctStates; + + private byte[] raw; + private byte[] compressed; + + /** + * Creates a new benchmark instance. + */ + public ChunkCompressionBenchmark() { + } + + /** + * Builds the serialised chunk and the compressed form of it. + * + * @throws IOException if the chunk cannot be serialised + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + ChunkColumn column = ChunkColumn.of(BenchmarkConstants.OVERWORLD_SECTIONS, this.distinctStates); + FakePaletteEntryResolver resolver = new FakePaletteEntryResolver(); + this.raw = ChunkPayloads.serialize(column, resolver, resolver); + this.compressed = this.compression.compress(this.raw); + } + + /** + * Compresses a whole chunk payload, which is the last stage of a chunk save. + * + * @return the compressed payload + * @throws IOException if the payload cannot be compressed + */ + @Benchmark + public byte[] compress() throws IOException { + return this.compression.compress(this.raw); + } + + /** + * Decompresses a whole chunk payload, which is the first stage of a chunk load. + * + * @return the uncompressed payload + * @throws IOException if the payload cannot be decompressed + */ + @Benchmark + public byte[] decompress() throws IOException { + return this.compression.decompress(this.compressed); + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/ChunkSaveComparisonBenchmark.java b/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/ChunkSaveComparisonBenchmark.java new file mode 100644 index 00000000..2e37b868 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/ChunkSaveComparisonBenchmark.java @@ -0,0 +1,404 @@ +package net.theevilreaper.aves.benchmark.anvil; + +import net.kyori.adventure.key.Key; +import net.kyori.adventure.nbt.BinaryTagIO; +import net.kyori.adventure.nbt.CompoundBinaryTag; +import net.minestom.server.MinecraftServer; +import net.minestom.server.instance.Chunk; +import net.minestom.server.instance.Instance; +import net.minestom.server.instance.Section; +import net.minestom.server.instance.anvil.AnvilLoader; +import net.minestom.server.instance.block.Block; +import net.theevilreaper.aves.instance.anvil.AvesAnvilLoader; +import net.theevilreaper.aves.instance.anvil.ChunkCompression; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +/** + * The {@link ChunkSaveComparisonBenchmark} class measures the whole chunk loader of Aves against + * the {@code AnvilLoader} of Minestom, on the same chunks and through the same registries. + *

+ * The loader of Minestom cannot be reached from a bare benchmark fork: its static fields read the + * biome registry and the block state count, so touching the class before a server exists fails in + * the class initialiser. The benchmark therefore starts a server in its trial setup, exactly as the + * light engine comparison does, and only then builds the chunks and the two loaders. + *

+ *

+ * The interesting axis is the amount of distinct block states a section holds, because that is + * where the two save paths differ structurally. Minestom deduplicates a palette entry with a linear + * search over an {@code IntArrayList}, so the cost of a section grows with the product of its block + * count and its palette size. Aves deduplicates through a hash map, so the cost grows with the + * block count alone. A single measurement point cannot tell those two apart, a series over the + * palette size can. + *

+ *

+ * The two loaders are measured as they ship, which includes their different compression levels: + * Minestom writes at the default level of the platform, Aves at + * {@link ChunkCompression#DEFAULT_LEVEL}. That difference is deliberate on the side of Aves and + * belongs to what a user of the loader gets, so hiding it would misrepresent both. It is also the + * one part of the result that is not about the loader structure, which is why the two calibration + * benchmarks below measure it separately: subtracting them from the save numbers leaves the part of + * the difference that the compression level does not explain. + *

+ *

+ * Every thread saves a chunk of its own, all of them inside the same region file. That is what a + * server does when it flushes a region, and it is the case in which the lock of the region file + * decides the throughput. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(value = 1, jvmArgsAppend = {"-Xms2g", "-Xmx2g"}) +@Warmup(iterations = 3, time = 2) +@Measurement(iterations = 5, time = 2) +public class ChunkSaveComparisonBenchmark { + + /** + * The amount of chunks the benchmark prepares, one per benchmark thread in rotation. + */ + private static final int CHUNK_COUNT = 32; + + /** + * The width of the chunk grid, chosen so all chunks land in the same region file. + */ + private static final int GRID_WIDTH = 8; + + /** + * The compression level the Adventure writer of Minestom uses for a chunk payload. + */ + private static final int MINESTOM_LEVEL = 6; + + private static final long SEED = 20260731L; + + private static final BinaryTagIO.Writer TAG_WRITER = BinaryTagIO.writer(); + + /** + * Hands every benchmark thread a chunk of its own. + */ + private static final AtomicInteger SLOTS = new AtomicInteger(); + + /** + * The amount of distinct block states a single section of the measured chunks holds. + *

+ * One is the uniform section both sides recognise and skip, which is the control point of the + * series. The values beyond it grow the palette, which is the axis the linear search of + * Minestom is expected to be sensitive to and the hash map of Aves is not. + *

+ */ + @Param({"1", "16", "64", "256", "1024"}) + public int distinctStates; + + private Path directory; + private Instance instance; + private List chunks; + private AnvilLoader minestomLoader; + private AvesAnvilLoader avesLoader; + private byte[] serialized; + + /** + * Creates a new benchmark instance. + */ + public ChunkSaveComparisonBenchmark() { + } + + /** + * Starts the server, builds the chunks and creates both loaders on separate world directories. + * + * @throws IOException if the world directories cannot be prepared + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + if (MinecraftServer.process() == null) { + MinecraftServer.init(); + } + + this.directory = Files.createTempDirectory("aves-save-comparison"); + Key dimension = Key.key("minecraft:overworld"); + Path minestomRoot = this.directory.resolve("minestom"); + Path avesRoot = this.directory.resolve("aves"); + Files.createDirectories(minestomRoot.resolve("dimensions/minecraft/overworld/region")); + Files.createDirectories(avesRoot.resolve("dimensions/minecraft/overworld/region")); + + this.minestomLoader = new AnvilLoader(minestomRoot, dimension); + this.avesLoader = new AvesAnvilLoader(avesRoot, dimension); + + this.instance = MinecraftServer.getInstanceManager().createInstanceContainer(); + int[] states = distinctStates(this.distinctStates); + this.chunks = new ArrayList<>(CHUNK_COUNT); + + for (int slot = 0; slot < CHUNK_COUNT; slot++) { + this.chunks.add(build(this.instance, chunkX(slot), chunkZ(slot), states, slot)); + } + + ByteArrayOutputStream target = new ByteArrayOutputStream(128 * 1024); + TAG_WRITER.writeNamed(Map.entry("", snapshotOf(this.chunks.getFirst())), target, BinaryTagIO.Compression.NONE); + this.serialized = target.toByteArray(); + } + + /** + * Removes the world directories of both loaders. + * + * @throws IOException if the loader of Aves cannot be closed + */ + @TearDown(Level.Trial) + public void tearDown() throws IOException { + this.avesLoader.close(); + + try (Stream entries = Files.walk(this.directory)) { + entries.sorted(Comparator.reverseOrder()).forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException ignored) { + // A leftover file in the temporary directory does not invalidate a measurement. + } + }); + } + } + + /** + * Saves a chunk through the loader of Minestom. + * + * @param slot the chunk this thread works on + */ + @Benchmark + public void minestomSave(ThreadSlot slot) { + this.minestomLoader.saveChunk(this.chunks.get(slot.slot)); + } + + /** + * Saves the same chunk through the loader of Aves. + * + * @param slot the chunk this thread works on + */ + @Benchmark + public void avesSave(ThreadSlot slot) { + this.avesLoader.saveChunk(this.chunks.get(slot.slot)); + } + + /** + * Compresses the serialised chunk at the level the loader of Aves ships with. + *

+ * The method measures no loader at all. It exists so the share of the save difference that is + * only the compression level can be subtracted from the two save numbers. + *

+ * + * @return the compressed payload + * @throws IOException if the payload cannot be compressed + */ + @Benchmark + public byte[] compressAvesLevel() throws IOException { + return ChunkCompression.ZLIB.compress(this.serialized, ChunkCompression.DEFAULT_LEVEL); + } + + /** + * Compresses the same serialised chunk at the level the loader of Minestom writes with. + * + * @return the compressed payload + * @throws IOException if the payload cannot be compressed + */ + @Benchmark + public byte[] compressMinestomLevel() throws IOException { + return ChunkCompression.ZLIB.compress(this.serialized, MINESTOM_LEVEL); + } + + /** + * Builds a chunk whose sections hold the given amount of distinct block states. + * + * @param instance the instance the chunk belongs to + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @param states the block state ids the sections are filled from + * @param seed an offset which keeps the chunks from being identical + * @return the created chunk + */ + private static Chunk build(Instance instance, int chunkX, int chunkZ, int[] states, int seed) { + Chunk chunk = instance.getChunkSupplier().createChunk(instance, chunkX, chunkZ); + Random random = new Random(SEED + seed); + byte[] light = new byte[2048]; + + for (Section section : chunk.getSections()) { + int[] values = shuffled(states, random); + section.blockPalette().setAll((x, y, z) -> values[(y << 8) | (z << 4) | x]); + random.nextBytes(light); + section.skyLight().set(light.clone()); + section.blockLight().set(new byte[2048]); + } + return chunk; + } + + /** + * Builds the value of every block of a section so the section holds exactly as many distinct + * states as the given array does. + * + * @param states the block state ids to spread over the section + * @param random the source of the shuffle + * @return the value of every block of the section + */ + private static int[] shuffled(int[] states, Random random) { + int[] values = new int[16 * 16 * 16]; + + for (int index = 0; index < values.length; index++) { + values[index] = states[index % states.length]; + } + for (int index = values.length - 1; index > 0; index--) { + int other = random.nextInt(index + 1); + int swap = values[index]; + values[index] = values[other]; + values[other] = swap; + } + return values; + } + + /** + * Collects the requested amount of distinct block state ids from the registry. + * + * @param wanted the amount of distinct block states to collect + * @return the collected block state ids + * @throws IllegalStateException if the registry holds fewer states than requested + */ + private static int[] distinctStates(int wanted) { + List collected = new ArrayList<>(wanted); + + for (Block block : Block.values()) { + for (Block state : block.possibleStates()) { + collected.add(state.stateId()); + + if (collected.size() >= wanted) { + break; + } + } + if (collected.size() >= wanted) { + break; + } + } + if (collected.size() < wanted) { + throw new IllegalStateException("The registry holds only " + collected.size() + " of " + wanted + " states"); + } + + int[] states = new int[wanted]; + + for (int index = 0; index < wanted; index++) { + states[index] = collected.get(index); + } + return states; + } + + /** + * Builds the chunk compound of the given chunk through the loader of Aves. + *

+ * The result is only used to obtain a realistic uncompressed payload for the two calibration + * benchmarks, which need the bytes and not the loader. + *

+ * + * @param chunk the chunk to describe + * @return the chunk data as the loader of Aves stores it + * @throws IOException if the chunk cannot be described + */ + private CompoundBinaryTag snapshotOf(Chunk chunk) throws IOException { + Path probeRoot = this.directory.resolve("probe"); + Files.createDirectories(probeRoot.resolve("dimensions/minecraft/overworld/region")); + + try (AvesAnvilLoader probe = new AvesAnvilLoader(probeRoot, Key.key("minecraft:overworld"))) { + probe.saveChunk(chunk); + } + Path region = probeRoot.resolve("dimensions/minecraft/overworld/region") + .resolve("r." + (chunk.getChunkX() >> 5) + "." + (chunk.getChunkZ() >> 5) + ".mca"); + + try (net.theevilreaper.aves.instance.anvil.RegionFile file = + net.theevilreaper.aves.instance.anvil.RegionFile.open(region)) { + net.theevilreaper.aves.instance.anvil.RegionFile.RawChunk raw = + file.readRaw(chunk.getChunkX(), chunk.getChunkZ()); + + if (raw == null) { + throw new IOException("The probe region file does not hold the chunk"); + } + return BinaryTagIO.unlimitedReader().read( + new java.io.ByteArrayInputStream(raw.decompress()), BinaryTagIO.Compression.NONE + ); + } + } + + /** + * Returns the x coordinate of the chunk which belongs to the given slot. + * + * @param slot the slot to resolve + * @return the absolute chunk x coordinate + */ + private static int chunkX(int slot) { + return slot % GRID_WIDTH; + } + + /** + * Returns the z coordinate of the chunk which belongs to the given slot. + * + * @param slot the slot to resolve + * @return the absolute chunk z coordinate + */ + private static int chunkZ(int slot) { + return slot / GRID_WIDTH; + } + + /** + * The {@link ThreadSlot} class assigns one of the prepared chunks to a benchmark thread. + *

+ * Two threads which save the same chunk would collide on the lock of that chunk, which is a + * different measurement than the one this benchmark is after. A server flushing a region saves + * different chunks into the same region file, which is what a slot per thread reproduces. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ + @State(Scope.Thread) + public static class ThreadSlot { + + /** + * The index of the chunk this thread works on. + */ + public int slot; + + /** + * Creates a new thread slot. + */ + public ThreadSlot() { + } + + /** + * Picks the chunk of this thread. + */ + @Setup(Level.Trial) + public void setUp() { + this.slot = SLOTS.getAndIncrement() % CHUNK_COUNT; + } + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/ChunkSaveStageBenchmark.java b/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/ChunkSaveStageBenchmark.java new file mode 100644 index 00000000..2e0a71d2 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/ChunkSaveStageBenchmark.java @@ -0,0 +1,191 @@ +package net.theevilreaper.aves.benchmark.anvil; + +import net.kyori.adventure.nbt.CompoundBinaryTag; +import net.theevilreaper.aves.benchmark.support.BenchmarkConstants; +import net.theevilreaper.aves.benchmark.support.ChunkColumn; +import net.theevilreaper.aves.benchmark.support.ChunkPayloads; +import net.theevilreaper.aves.benchmark.support.FakePaletteEntryResolver; +import net.theevilreaper.aves.instance.anvil.ChunkCompression; +import net.theevilreaper.aves.instance.anvil.RegionFile; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +/** + * The {@link ChunkSaveStageBenchmark} class splits a whole chunk save into the three stages it + * consists of and measures each of them on its own. + *

+ * The loader is built around one claim: only the snapshot stage holds the read lock of the chunk + * and only the transfer stage holds the lock of the region file, while the codec stage, which is + * the expensive one, runs without any lock at all. That claim is structural in the source and this + * benchmark is what turns it into a number. Comparing the three stages shows how much of a save + * really happens outside of a lock. + *

+ *
    + *
  • {@code snapshot} copies the arrays of every section. The real loader performs this while it + * holds the read lock of the chunk, so a game thread waits for exactly this stage.
    • + *
  • {@code codec} builds the palettes, packs the indices, serialises the NBT and compresses the + * result. No lock is held here.
    • + *
  • {@code transfer} hands the finished bytes to the region file. The region lock is held for + * the sector allocation and the header update inside this stage.
    • + *
  • {@code full} performs all three so the sum of the parts can be checked against the whole.
    • + *
+ *

+ * The chunk is not a Minestom chunk. A Minestom section needs a started server and its registries, + * which would put registry time into the codec stage and hide the very thing this benchmark is + * supposed to isolate. The section arrays are therefore plain arrays of state ids and the resolver + * is a fake one. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(value = 1, jvmArgsAppend = {"-Xms1g", "-Xmx1g"}) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +public class ChunkSaveStageBenchmark { + + private static final int CHUNK_X = 3; + private static final int CHUNK_Z = 11; + + /** + * The amount of distinct block states a single section of the chunk holds. + */ + @Param({"8", "200"}) + public int distinctStates; + + private ChunkColumn column; + private FakePaletteEntryResolver resolver; + private ChunkColumn snapshot; + private byte[] compressed; + private Path directory; + private RegionFile region; + + /** + * Creates a new benchmark instance. + */ + public ChunkSaveStageBenchmark() { + } + + /** + * Builds the chunk, the intermediate results of every stage and the temporary region file. + *

+ * Every stage benchmark needs the output of the previous one as its input. Preparing those here + * keeps each measured method restricted to the stage it is named after. + *

+ * + * @throws IOException if the region file or the payload cannot be prepared + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + this.column = ChunkColumn.of(BenchmarkConstants.OVERWORLD_SECTIONS, this.distinctStates); + this.resolver = new FakePaletteEntryResolver(); + this.snapshot = this.column.copy(); + this.compressed = ChunkCompression.ZLIB.compress( + ChunkPayloads.serialize(this.snapshot, this.resolver, this.resolver) + ); + + this.directory = Files.createTempDirectory("aves-save-benchmark"); + this.region = RegionFile.open(this.directory.resolve("r.0.0.mca")); + this.region.writeRaw(CHUNK_X, CHUNK_Z, ChunkCompression.ZLIB, this.compressed); + } + + /** + * Closes the region file and removes the temporary directory again. + * + * @throws IOException if the temporary files cannot be removed + */ + @TearDown(Level.Trial) + public void tearDown() throws IOException { + this.region.close(); + + try (Stream entries = Files.walk(this.directory)) { + entries.sorted(Comparator.reverseOrder()).forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException exception) { + throw new java.io.UncheckedIOException(exception); + } + }); + } + } + + /** + * Copies every section array of the chunk. + * This is the only stage the real loader performs under the read lock of the chunk. + * + * @return the copied chunk + */ + @Benchmark + public ChunkColumn snapshot() { + return this.column.copy(); + } + + /** + * Builds the palettes, the NBT and the compressed payload from an already copied chunk. + * This stage runs without holding any lock. + * + * @return the compressed payload of the chunk + * @throws IOException if the payload cannot be built + */ + @Benchmark + public byte[] codec() throws IOException { + return ChunkCompression.ZLIB.compress(ChunkPayloads.serialize(this.snapshot, this.resolver, this.resolver)); + } + + /** + * Builds only the NBT of an already copied chunk, without compressing it. + * Comparing this against {@link #codec()} splits the codec stage into its palette part and its + * compression part. + * + * @return the chunk data of the chunk + */ + @Benchmark + public CompoundBinaryTag codecWithoutCompression() { + return ChunkPayloads.encode(this.snapshot, this.resolver, this.resolver); + } + + /** + * Hands an already compressed payload to the region file. + * The region lock is held inside this stage. + * + * @throws IOException if the payload cannot be written + */ + @Benchmark + public void transfer() throws IOException { + this.region.writeRaw(CHUNK_X, CHUNK_Z, ChunkCompression.ZLIB, this.compressed); + } + + /** + * Performs a whole chunk save from the snapshot to the written bytes. + * + * @throws IOException if the chunk cannot be saved + */ + @Benchmark + public void full() throws IOException { + ChunkColumn copy = this.column.copy(); + byte[] payload = ChunkCompression.ZLIB.compress(ChunkPayloads.serialize(copy, this.resolver, this.resolver)); + this.region.writeRaw(CHUNK_X, CHUNK_Z, ChunkCompression.ZLIB, payload); + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/PaletteDataBenchmark.java b/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/PaletteDataBenchmark.java new file mode 100644 index 00000000..242b6ed7 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/PaletteDataBenchmark.java @@ -0,0 +1,108 @@ +package net.theevilreaper.aves.benchmark.anvil; + +import net.theevilreaper.aves.benchmark.support.BenchmarkConstants; +import net.theevilreaper.aves.benchmark.support.SectionStates; +import net.theevilreaper.aves.instance.anvil.PaletteData; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +/** + * The {@link PaletteDataBenchmark} class measures the conversion of a section between its raw state + * ids and its palette representation. + *

+ * The encoding is the dominant part of a chunk save. It walks the whole section once to collect the + * distinct states into a palette and once more to pack the indices which reference them. The + * collection runs through a hash map, which is why the amount of distinct states of a section is + * the parameter of this benchmark and not the section size. + *

+ *

+ * The chosen amounts describe the sections a real world holds. One distinct state is a section of + * pure air or pure stone, which is the majority of every world and which the encoder answers + * without packing anything at all. Eight is an ordinary underground section, sixty four a surface + * section with vegetation, and two hundred a heavily built section which already needs eight bits + * per entry. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(value = 2, jvmArgsAppend = {"-Xms512m", "-Xmx512m"}) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +public class PaletteDataBenchmark { + + /** + * The amount of distinct block states a single section holds. + */ + @Param({"1", "8", "64", "200"}) + public int distinctStates; + + private int[] values; + private PaletteData encoded; + + /** + * Builds the raw states of the section and the already encoded representation of it. + */ + @Setup(Level.Trial) + public void setUp() { + this.values = SectionStates.distinct(BenchmarkConstants.BLOCK_ENTRIES, this.distinctStates, 1); + this.encoded = PaletteData.encode(this.values, BenchmarkConstants.BLOCK_PALETTE_MIN_BITS); + } + + /** + * Creates a new benchmark instance. + */ + public PaletteDataBenchmark() { + } + + /** + * Collects the palette of a section and packs the indices which reference it. + * This is the work a chunk save performs once per section. + * + * @return the palette representation of the section + */ + @Benchmark + public PaletteData encode() { + return PaletteData.encode(this.values, BenchmarkConstants.BLOCK_PALETTE_MIN_BITS); + } + + /** + * Resolves every entry of a section through its palette. + * This is the work a chunk load performs once per section. + * + * @return the state id of every block of the section + * @throws IOException if a packed index does not address a palette entry + */ + @Benchmark + public int[] unpack() throws IOException { + return this.encoded.unpack(); + } + + /** + * Encodes and resolves a section again, which is what a load followed by a save performs. + * + * @return the state id of every block of the section + * @throws IOException if a packed index does not address a palette entry + */ + @Benchmark + public int[] roundTrip() throws IOException { + return PaletteData.encode(this.values, BenchmarkConstants.BLOCK_PALETTE_MIN_BITS).unpack(); + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/RegionFileBenchmark.java b/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/RegionFileBenchmark.java new file mode 100644 index 00000000..418c7b08 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/RegionFileBenchmark.java @@ -0,0 +1,149 @@ +package net.theevilreaper.aves.benchmark.anvil; + +import net.theevilreaper.aves.benchmark.support.BenchmarkConstants; +import net.theevilreaper.aves.benchmark.support.ChunkColumn; +import net.theevilreaper.aves.benchmark.support.ChunkPayloads; +import net.theevilreaper.aves.benchmark.support.FakePaletteEntryResolver; +import net.theevilreaper.aves.instance.anvil.ChunkCompression; +import net.theevilreaper.aves.instance.anvil.RegionFile; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +/** + * The {@link RegionFileBenchmark} class measures the byte transfer of a region file. + *

+ * The measured operations move an already compressed payload. That is the whole point of the split + * the region file enforces: the caller compresses and decompresses on its own, and only the + * transfer of the finished bytes touches the sector allocation and the header, which are the parts + * the internal lock protects. + *

+ *

+ * Reading uses positional channel operations and takes no lock, so the read benchmark describes a + * path several threads can walk at the same time. What it adds on top of the raw transfer are the + * two reads of the version counter of the chunk which tell the reader whether a writer moved the + * bytes underneath it. Writing takes the lock for the allocation and the header update, so the write + * benchmark describes the part of a save which really serialises between threads. + *

+ *

+ * The numbers depend heavily on the file system and on the page cache of the machine. A run on a + * warm cache measures almost no device time at all, which is realistic for a server that saves the + * same chunks over and over, but it is not a measurement of the storage device. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(value = 1, jvmArgsAppend = {"-Xms512m", "-Xmx512m"}) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 1) +public class RegionFileBenchmark { + + private static final int CHUNK_X = 5; + private static final int CHUNK_Z = 7; + + private Path directory; + private RegionFile region; + private byte[] payload; + + /** + * Creates a new benchmark instance. + */ + public RegionFileBenchmark() { + } + + /** + * Creates a temporary region file and stores one chunk in it. + * The stored chunk is what the read benchmark reads back. + * + * @throws IOException if the region file cannot be prepared + */ + @Setup(Level.Trial) + public void setUp() throws IOException { + this.directory = Files.createTempDirectory("aves-region-benchmark"); + this.region = RegionFile.open(this.directory.resolve("r.0.0.mca")); + + ChunkColumn column = ChunkColumn.of(BenchmarkConstants.OVERWORLD_SECTIONS, 64); + FakePaletteEntryResolver resolver = new FakePaletteEntryResolver(); + this.payload = ChunkCompression.ZLIB.compress(ChunkPayloads.serialize(column, resolver, resolver)); + this.region.writeRaw(CHUNK_X, CHUNK_Z, ChunkCompression.ZLIB, this.payload); + } + + /** + * Closes the region file and removes the temporary directory again. + * + * @throws IOException if the temporary files cannot be removed + */ + @TearDown(Level.Trial) + public void tearDown() throws IOException { + this.region.close(); + + try (Stream entries = Files.walk(this.directory)) { + entries.sorted(Comparator.reverseOrder()).forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException exception) { + throw new java.io.UncheckedIOException(exception); + } + }); + } + } + + /** + * Writes an already compressed chunk payload into the region file. + *

+ * The chunk is written to the same coordinate every time. The allocator frees the previous + * sectors after it reserved the new ones, so the file alternates between two sector ranges and + * cannot grow without bound over a long run. + *

+ * + * @throws IOException if the payload cannot be written + */ + @Benchmark + public void writeRaw() throws IOException { + this.region.writeRaw(CHUNK_X, CHUNK_Z, ChunkCompression.ZLIB, this.payload); + } + + /** + * Reads the raw payload of a chunk without decompressing it. + * + * @return the raw chunk which was read + * @throws IOException if the payload cannot be read + */ + @Benchmark + public RegionFile.RawChunk readRaw() throws IOException { + return this.region.readRaw(CHUNK_X, CHUNK_Z); + } + + /** + * Writes and reads a chunk again, which is the full byte transfer of a save followed by a load. + * + * @return the raw chunk which was read + * @throws IOException if the payload cannot be transferred + */ + @Benchmark + public RegionFile.RawChunk roundTrip() throws IOException { + this.region.writeRaw(CHUNK_X, CHUNK_Z, ChunkCompression.ZLIB, this.payload); + return this.region.readRaw(CHUNK_X, CHUNK_Z); + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/package-info.java b/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/package-info.java new file mode 100644 index 00000000..bff5b00d --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/anvil/package-info.java @@ -0,0 +1,13 @@ +/** + * Contains the benchmarks of the Anvil chunk loader. + *

+ * The benchmarks cover the bit packing, the palette encoding, the compression and the byte transfer + * of a region file, and one benchmark which splits a whole chunk save into those stages so the cost + * inside a lock can be compared against the cost outside of it. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +package net.theevilreaper.aves.benchmark.anvil; diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/light/ChunkLightPropagatorBenchmark.java b/src/jmh/java/net/theevilreaper/aves/benchmark/light/ChunkLightPropagatorBenchmark.java new file mode 100644 index 00000000..a5437664 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/light/ChunkLightPropagatorBenchmark.java @@ -0,0 +1,119 @@ +package net.theevilreaper.aves.benchmark.light; + +import net.theevilreaper.aves.benchmark.support.FakeBlockLightSource; +import net.theevilreaper.aves.benchmark.support.SectionStates; +import net.theevilreaper.aves.instance.light.ChunkLightPropagator; +import net.theevilreaper.aves.instance.light.LightNibbles; +import net.theevilreaper.aves.instance.light.SectionOpacity; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * The {@link ChunkLightPropagatorBenchmark} class measures the light propagation over a whole chunk + * column. + *

+ * A propagation which stops at a section border produces a seam every sixteen blocks, so the chunk + * propagator treats every section of a chunk as one column and lets the search cross their borders. + * That makes the amount of sections the parameter which decides the size of the search space: four + * sections are a flat map, sixteen a shallow world and twenty four the full height of a modern + * overworld. + *

+ *

+ * Two searches are measured because the engine performs two of them per chunk. Block light starts + * at the emitting blocks and is bounded by their amount, while sky light starts at every block that + * sees the open sky, which in an open column is nearly the whole chunk. The two therefore behave + * very differently and reporting only one of them would describe half of the work. + *

+ *

+ * The propagator instance is reused across invocations. It sizes its buffers for the largest column + * it has seen and keeps them, so a fresh instance per invocation would measure the buffer + * allocation and not the search. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(value = 2, jvmArgsAppend = {"-Xms512m", "-Xmx512m"}) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +public class ChunkLightPropagatorBenchmark { + + /** + * The amount of sections the chunk holds. + */ + @Param({"4", "16", "24"}) + public int sectionCount; + + /** + * The amount of light emitting blocks a single section of the chunk holds. + */ + @Param({"1", "8"}) + public int lightSourcesPerSection; + + private ChunkLightPropagator propagator; + private List sections; + + /** + * Creates a new benchmark instance. + */ + public ChunkLightPropagatorBenchmark() { + } + + /** + * Builds the opacity table of every section of the chunk and warms the buffers of the + * propagator so the first measured invocation does not pay for their allocation. + */ + @Setup(Level.Trial) + public void setUp() { + FakeBlockLightSource source = new FakeBlockLightSource(); + this.propagator = new ChunkLightPropagator(); + this.sections = new ArrayList<>(this.sectionCount); + + for (int section = 0; section < this.sectionCount; section++) { + // A quarter of the blocks of the lower half are solid, which gives the search something + // to stop at, while the upper half stays open so the sky light really reaches downwards. + int occlusion = section < this.sectionCount / 2 ? 25 : 0; + this.sections.add(SectionOpacity.of(SectionStates.lit(this.lightSourcesPerSection, occlusion), source)); + } + this.propagator.propagate(this.sections); + this.propagator.propagateSky(this.sections); + } + + /** + * Spreads the light of every emitting block of the chunk through the whole column. + * + * @return the calculated light of every section + */ + @Benchmark + public List propagate() { + return this.propagator.propagate(this.sections); + } + + /** + * Lets the sky light fall into the chunk and spread from where it is stopped. + * + * @return the calculated sky light of every section + */ + @Benchmark + public List propagateSky() { + return this.propagator.propagateSky(this.sections); + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/light/LightNibblesBenchmark.java b/src/jmh/java/net/theevilreaper/aves/benchmark/light/LightNibblesBenchmark.java new file mode 100644 index 00000000..e85bbfd1 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/light/LightNibblesBenchmark.java @@ -0,0 +1,189 @@ +package net.theevilreaper.aves.benchmark.light; + +import net.theevilreaper.aves.instance.light.LightNibbles; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.concurrent.TimeUnit; + +/** + * The {@link LightNibblesBenchmark} class measures the nibble storage of a light section. + *

+ * The storage has two shapes. A section in which every block carries the same level keeps no array + * at all and answers every read from a single field, and a section whose levels differ keeps a + * {@value LightNibbles#ARRAY_LENGTH} byte array and has to shift the requested nibble out of it. + * The first shape is the common one, because most sections of a world are either completely dark or + * completely lit by the sky, and this benchmark is what shows what that shortcut is worth. + *

+ *

+ * Every measured method sweeps the whole section instead of touching a single block. A single + * nibble read is a handful of instructions, which is below what a harness can separate from its own + * overhead. The reported time is therefore the time of a full sweep over 4096 blocks and the cost + * of a single access follows from dividing it. + *

+ *

+ * That division only holds for the sweeps over an allocated section. The two sweeps over a uniform + * section are collapsed by the compiler because their loop body does not depend on the coordinates, + * and their numbers are lower bounds rather than per access costs. The details are on + * {@link #sweep(LightNibbles, Blackhole)} and on {@link #setUniformUnchanged()}. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(value = 2, jvmArgsAppend = {"-Xms512m", "-Xmx512m"}) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +public class LightNibblesBenchmark { + + private LightNibbles uniform; + private LightNibbles allocated; + private byte[] stored; + + /** + * Creates a new benchmark instance. + */ + public LightNibblesBenchmark() { + } + + /** + * Builds one section of each shape. + * The allocated one receives a level pattern which no compaction can collapse again. + */ + @Setup(Level.Trial) + public void setUp() { + this.uniform = LightNibbles.uniform(LightNibbles.MAX_LEVEL); + this.allocated = LightNibbles.uniform(0); + + for (int y = 0; y < LightNibbles.DIMENSION; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + this.allocated.set(x, y, z, (x + y + z) & LightNibbles.MAX_LEVEL); + } + } + } + this.stored = this.allocated.toDenseArray(); + } + + /** + * Reads every block of a section which carries a single level everywhere. + * + * @param hole the sink which receives every level that was read + */ + @Benchmark + public void getUniform(Blackhole hole) { + sweep(this.uniform, hole); + } + + /** + * Reads every block of a section which keeps a full nibble array. + * + * @param hole the sink which receives every level that was read + */ + @Benchmark + public void getAllocated(Blackhole hole) { + sweep(this.allocated, hole); + } + + /** + * Writes the level a uniform section already carries to every one of its blocks. + * The section stays uniform and no array is ever allocated, which is the shortcut a propagation + * of a dark section relies on. + *

+ * Expect a number close to zero. Every one of these writes returns without touching anything, + * the compiler can prove that, and it removes the loop. The result is therefore a statement + * about how cheap the shortcut can get and not a per call cost. It is kept because a change + * which accidentally makes this path allocate would show up here immediately. + *

+ * + * @return the section which was written to + */ + @Benchmark + public LightNibbles setUniformUnchanged() { + LightNibbles light = LightNibbles.uniform(0); + + for (int y = 0; y < LightNibbles.DIMENSION; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + light.set(x, y, z, 0); + } + } + } + return light; + } + + /** + * Writes a differing level to every block of a fresh uniform section. + * The first write allocates the array, which makes this the exact path a propagation takes when + * it transfers its result into a section. + * + * @return the section which was written to + */ + @Benchmark + public LightNibbles setAllocating() { + LightNibbles light = LightNibbles.uniform(0); + + for (int y = 0; y < LightNibbles.DIMENSION; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + light.set(x, y, z, (x + y + z) & LightNibbles.MAX_LEVEL); + } + } + } + return light; + } + + /** + * Reads a stored section back from its byte form, which is what a chunk load performs. + * + * @return the section which was read + */ + @Benchmark + public LightNibbles ofArray() { + return LightNibbles.of(this.stored); + } + + /** + * Reads the level of every block of the given section into the given sink. + *

+ * Every single level is handed to the sink instead of being summed up, so the reads of a + * section which keeps an array cannot be folded away. + *

+ *

+ * A uniform section is a different matter and the sink does not save it either. Such a section + * answers every read from one field regardless of the coordinates, which makes the whole loop + * body loop invariant, and the compiler hoists it out and drops the empty loop. Both blackhole + * modes behave the same way here, which was verified with + * {@code -Djmh.blackhole.autoDetect=false}. The number {@link #getUniform(Blackhole)} reports is + * therefore a lower bound and not the cost of 4096 reads: it says that reading a uniform + * section can collapse to a single field read, which is the property the shortcut exists for. + * Only {@link #getAllocated(Blackhole)} divided by 4096 is a per access cost. + *

+ * + * @param light the section to read + * @param hole the sink which receives every level that was read + */ + private static void sweep(LightNibbles light, Blackhole hole) { + for (int y = 0; y < LightNibbles.DIMENSION; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + hole.consume(light.get(x, y, z)); + } + } + } + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/light/LightPropagatorBenchmark.java b/src/jmh/java/net/theevilreaper/aves/benchmark/light/LightPropagatorBenchmark.java new file mode 100644 index 00000000..c41ddfe7 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/light/LightPropagatorBenchmark.java @@ -0,0 +1,101 @@ +package net.theevilreaper.aves.benchmark.light; + +import net.theevilreaper.aves.benchmark.support.FakeBlockLightSource; +import net.theevilreaper.aves.benchmark.support.SectionStates; +import net.theevilreaper.aves.instance.light.LightNibbles; +import net.theevilreaper.aves.instance.light.LightPropagator; +import net.theevilreaper.aves.instance.light.SectionOpacity; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.concurrent.TimeUnit; + +/** + * The {@link LightPropagatorBenchmark} class measures the block light propagation of a single + * section. + *

+ * The propagation is a breadth-first search whose cost grows with the amount of positions it + * queues, so the amount of light sources of the section is the parameter that matters. A section + * without any source is answered without a search at all, which is the case for the overwhelming + * majority of the sections of a world and which is exactly why the shortcut exists. + *

+ *

+ * The share of solid blocks is the second parameter. Solid blocks stop the search early, so a + * section full of them performs less work than an open one with the same amount of sources. A + * benchmark which only measured an empty section would therefore report the worst case and call it + * the normal one. + *

+ *

+ * The propagator instance is created once and reused across every invocation. That is how the class + * is meant to be used, because it keeps its working buffers between runs, and creating a fresh one + * per invocation would measure two array allocations instead of the search. + *

+ *

+ * The opacity table is built in the setup. Building it inside the measured method would make this + * benchmark a duplicate of {@link SectionOpacityBenchmark}. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(value = 2, jvmArgsAppend = {"-Xms512m", "-Xmx512m"}) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +public class LightPropagatorBenchmark { + + /** + * The amount of light emitting blocks the section holds. + */ + @Param({"0", "1", "8", "64"}) + public int lightSources; + + /** + * The share of solid blocks of the section in percent. + */ + @Param({"0", "25"}) + public int occlusionPercent; + + private LightPropagator propagator; + private SectionOpacity opacity; + + /** + * Creates a new benchmark instance. + */ + public LightPropagatorBenchmark() { + } + + /** + * Builds the opacity table of the section and the propagator which walks it. + */ + @Setup(Level.Trial) + public void setUp() { + this.propagator = new LightPropagator(); + this.opacity = SectionOpacity.of( + SectionStates.lit(this.lightSources, this.occlusionPercent), new FakeBlockLightSource() + ); + } + + /** + * Spreads the light of every emitting block through the section. + * + * @return the calculated light of the section + */ + @Benchmark + public LightNibbles propagate() { + return this.propagator.propagate(this.opacity); + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/light/SectionOpacityBenchmark.java b/src/jmh/java/net/theevilreaper/aves/benchmark/light/SectionOpacityBenchmark.java new file mode 100644 index 00000000..c605a7d3 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/light/SectionOpacityBenchmark.java @@ -0,0 +1,105 @@ +package net.theevilreaper.aves.benchmark.light; + +import net.theevilreaper.aves.benchmark.support.BenchmarkConstants; +import net.theevilreaper.aves.benchmark.support.FakeBlockLightSource; +import net.theevilreaper.aves.benchmark.support.SectionStates; +import net.theevilreaper.aves.instance.light.BlockLightSource; +import net.theevilreaper.aves.instance.light.SectionOpacity; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.concurrent.TimeUnit; + +/** + * The {@link SectionOpacityBenchmark} class measures the construction of the opacity table of a + * section. + *

+ * The table exists because a breadth-first search reaches the same block from up to six directions + * and would otherwise ask the registry for its properties every single time. The table asks once + * per distinct block state and answers from two arrays afterwards. This benchmark is what shows + * what that caching is worth. + *

+ *

+ * Two parameters describe the two things the cost depends on. The amount of distinct states decides + * how often the cache misses, and the resolve cost describes how expensive a single miss is. The + * cost is a parameter because the real source is a registry lookup and not an arithmetic + * expression. A source which answers instantly would make the hash map behind the cache look like + * pure overhead, which is the opposite of what happens on a running server. + *

+ *
    + *
  • A resolve cost of zero measures the table construction itself, so the cost of the hash map + * and of the two array writes per block.
    • + *
  • A resolve cost above zero measures what the cache saves. Seven resolutions happen per + * distinct state, one per face plus the emission, instead of seven per block.
    • + *
+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@Fork(value = 2, jvmArgsAppend = {"-Xms512m", "-Xmx512m"}) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 5, time = 1) +public class SectionOpacityBenchmark { + + /** + * The amount of distinct block states the section holds. + */ + @Param({"1", "8", "64", "200"}) + public int distinctStates; + + /** + * The amount of work tokens a single resolution of a block state burns. + * Zero measures the table itself, a higher value measures what the caching saves. + */ + @Param({"0", "50"}) + public int resolveCost; + + private int[] stateIds; + private BlockLightSource source; + + /** + * Creates a new benchmark instance. + */ + public SectionOpacityBenchmark() { + } + + /** + * Builds the states of the section and the source which describes them. + *

+ * The states start above {@link FakeBlockLightSource#FILLER_BASE} so every one of them behaves + * like air. The benchmark measures how often a state is resolved, not what the answer is, and a + * mixture of solid and transparent blocks would only add noise to that. + *

+ */ + @Setup(Level.Trial) + public void setUp() { + this.stateIds = SectionStates.distinct( + BenchmarkConstants.BLOCK_ENTRIES, this.distinctStates, FakeBlockLightSource.FILLER_BASE + ); + this.source = new FakeBlockLightSource(this.resolveCost); + } + + /** + * Builds the opacity table of a whole section. + * + * @return the created table + */ + @Benchmark + public SectionOpacity of() { + return SectionOpacity.of(this.stateIds, this.source); + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/light/package-info.java b/src/jmh/java/net/theevilreaper/aves/benchmark/light/package-info.java new file mode 100644 index 00000000..7f7727e0 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/light/package-info.java @@ -0,0 +1,14 @@ +/** + * Contains the benchmarks of the light engine. + *

+ * The benchmarks cover the nibble storage of a light section, the construction of the opacity table + * which the propagation reads, and the propagation itself for a single section and for a whole + * chunk column. None of them starts a server, because a registry lookup would otherwise dominate + * every measurement. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +package net.theevilreaper.aves.benchmark.light; diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/support/BenchmarkConstants.java b/src/jmh/java/net/theevilreaper/aves/benchmark/support/BenchmarkConstants.java new file mode 100644 index 00000000..896470d0 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/support/BenchmarkConstants.java @@ -0,0 +1,57 @@ +package net.theevilreaper.aves.benchmark.support; + +/** + * The {@link BenchmarkConstants} class holds the layout constants the benchmarks share. + *

+ * The values mirror the constants of the Minestom palette which the loader uses at runtime, but + * they are repeated here on purpose. Pulling Minestom onto the benchmark classpath would drag a + * registry and a server lifecycle into a harness that is supposed to measure the code of this + * library alone. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +public final class BenchmarkConstants { + + /** + * The amount of block entries a single section holds. + * This mirrors {@code Palette.BLOCK_DIMENSION} cubed. + */ + public static final int BLOCK_ENTRIES = 16 * 16 * 16; + + /** + * The amount of biome entries a single section holds. + * This mirrors {@code Palette.BIOME_DIMENSION} cubed. + */ + public static final int BIOME_ENTRIES = 4 * 4 * 4; + + /** + * The smallest amount of bits a block palette entry occupies. + * This mirrors {@code Palette.BLOCK_PALETTE_MIN_BITS}. + */ + public static final int BLOCK_PALETTE_MIN_BITS = 4; + + /** + * The smallest amount of bits a biome palette entry occupies. + * This mirrors {@code Palette.BIOME_PALETTE_MIN_BITS}. + */ + public static final int BIOME_PALETTE_MIN_BITS = 1; + + /** + * The amount of sections a chunk of a full height overworld holds. + */ + public static final int OVERWORLD_SECTIONS = 24; + + /** + * The seed every generator uses so two runs of the same benchmark see the same input. + */ + public static final long SEED = 0x5DEECE66DL; + + /** + * Blocks the creation of an instance because the class only holds constants. + */ + private BenchmarkConstants() { + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/support/ChunkColumn.java b/src/jmh/java/net/theevilreaper/aves/benchmark/support/ChunkColumn.java new file mode 100644 index 00000000..a30e89c0 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/support/ChunkColumn.java @@ -0,0 +1,92 @@ +package net.theevilreaper.aves.benchmark.support; + +import java.util.Random; + +/** + * The {@link ChunkColumn} record holds the raw arrays of a chunk in the shape the save path reads + * them from a Minestom section. + *

+ * The loader copies exactly these arrays while it holds the read lock of the chunk and converts + * them afterwards. Modelling the chunk as plain arrays lets the benchmark reproduce that split + * without a running server. + *

+ * + * @param blockStates the state id of every block of every section + * @param biomes the biome id of every biome cell of every section + * @param skyLight the stored sky light of every section + * @param blockLight the stored block light of every section + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +public record ChunkColumn(int[][] blockStates, int[][] biomes, byte[][] skyLight, byte[][] blockLight) { + + private static final int LIGHT_BYTES = BenchmarkConstants.BLOCK_ENTRIES / 2; + + /** + * Builds a chunk whose sections hold the given amount of distinct block states. + * + * @param sectionCount the amount of sections the chunk holds + * @param distinctStates the amount of distinct block states a single section holds + * @return the created chunk + * @throws IllegalArgumentException if the chunk holds no section + */ + public static ChunkColumn of(int sectionCount, int distinctStates) { + if (sectionCount <= 0) { + throw new IllegalArgumentException("A chunk has to hold at least one section but held " + sectionCount); + } + + int[][] blockStates = new int[sectionCount][]; + int[][] biomes = new int[sectionCount][]; + byte[][] skyLight = new byte[sectionCount][]; + byte[][] blockLight = new byte[sectionCount][]; + Random random = new Random(BenchmarkConstants.SEED); + + for (int section = 0; section < sectionCount; section++) { + // Every section starts its states at its own offset so the whole chunk holds a + // realistic amount of distinct states instead of repeating one palette everywhere. + blockStates[section] = SectionStates.distinct( + BenchmarkConstants.BLOCK_ENTRIES, distinctStates, 1 + section * distinctStates + ); + biomes[section] = SectionStates.distinct(BenchmarkConstants.BIOME_ENTRIES, Math.min(distinctStates, 4), 1); + skyLight[section] = new byte[LIGHT_BYTES]; + blockLight[section] = new byte[LIGHT_BYTES]; + random.nextBytes(skyLight[section]); + } + return new ChunkColumn(blockStates, biomes, skyLight, blockLight); + } + + /** + * Returns the amount of sections the chunk holds. + * + * @return the amount of sections + */ + public int sectionCount() { + return this.blockStates.length; + } + + /** + * Copies every array of the chunk. + *

+ * This is the work the loader performs while it holds the read lock of the chunk. Everything + * that follows runs on the copy and therefore outside of that lock. + *

+ * + * @return a chunk which shares no array with this one + */ + public ChunkColumn copy() { + int sectionCount = sectionCount(); + int[][] states = new int[sectionCount][]; + int[][] biomeCopy = new int[sectionCount][]; + byte[][] sky = new byte[sectionCount][]; + byte[][] block = new byte[sectionCount][]; + + for (int section = 0; section < sectionCount; section++) { + states[section] = this.blockStates[section].clone(); + biomeCopy[section] = this.biomes[section].clone(); + sky[section] = this.skyLight[section].clone(); + block[section] = this.blockLight[section].clone(); + } + return new ChunkColumn(states, biomeCopy, sky, block); + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/support/ChunkPayloads.java b/src/jmh/java/net/theevilreaper/aves/benchmark/support/ChunkPayloads.java new file mode 100644 index 00000000..9a04da6a --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/support/ChunkPayloads.java @@ -0,0 +1,114 @@ +package net.theevilreaper.aves.benchmark.support; + +import net.kyori.adventure.nbt.BinaryTagIO; +import net.kyori.adventure.nbt.BinaryTagTypes; +import net.kyori.adventure.nbt.CompoundBinaryTag; +import net.kyori.adventure.nbt.ListBinaryTag; +import net.theevilreaper.aves.instance.anvil.PaletteData; +import net.theevilreaper.aves.instance.anvil.PaletteEntryResolver; +import net.theevilreaper.aves.instance.anvil.SectionCodec; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.Map; + +/** + * The {@link ChunkPayloads} class turns a {@link ChunkColumn} into the NBT and the bytes a region + * file stores. + *

+ * The conversion follows the save path of the loader step by step. It builds one palette container + * per section, wraps them into the chunk compound the format defines and serialises the result + * without compression. A benchmark can therefore compress the very same bytes a real save would + * hand to the region file. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +public final class ChunkPayloads { + + private static final BinaryTagIO.Writer TAG_WRITER = BinaryTagIO.writer(); + private static final int INITIAL_BUFFER = 64 * 1024; + + /** + * Blocks the creation of an instance because the class only holds converters. + */ + private ChunkPayloads() { + } + + /** + * Builds the chunk compound of the given chunk. + * + * @param column the chunk to describe + * @param blockResolver the resolver which names the block states + * @param biomeResolver the resolver which names the biomes + * @return the chunk data of the chunk + */ + public static CompoundBinaryTag encode(ChunkColumn column, PaletteEntryResolver blockResolver, PaletteEntryResolver biomeResolver) { + ListBinaryTag.Builder sections = ListBinaryTag.builder(BinaryTagTypes.COMPOUND); + + for (int section = 0; section < column.sectionCount(); section++) { + sections.add(encodeSection(column, section, blockResolver, biomeResolver)); + } + + return CompoundBinaryTag.builder() + .putInt("DataVersion", 4189) + .putInt("xPos", 0) + .putInt("zPos", 0) + .putInt("yPos", -4) + .putString("Status", "minecraft:full") + .putLong("LastUpdate", 0L) + .put("sections", sections.build()) + .put("block_entities", ListBinaryTag.builder(BinaryTagTypes.COMPOUND).build()) + .build(); + } + + /** + * Builds the data of a single section. + * + * @param column the chunk which holds the section + * @param section the index of the section inside the chunk + * @param blockResolver the resolver which names the block states + * @param biomeResolver the resolver which names the biomes + * @return the data of the section + */ + private static CompoundBinaryTag encodeSection(ChunkColumn column, int section, PaletteEntryResolver blockResolver, PaletteEntryResolver biomeResolver) { + PaletteData blocks = PaletteData.encode(column.blockStates()[section], BenchmarkConstants.BLOCK_PALETTE_MIN_BITS); + PaletteData biomes = PaletteData.encode(column.biomes()[section], BenchmarkConstants.BIOME_PALETTE_MIN_BITS); + + return CompoundBinaryTag.builder() + .putByte("Y", (byte) (section - 4)) + .put("block_states", SectionCodec.encode(blocks, blockResolver)) + .put("biomes", SectionCodec.encodeBiomes(biomes, biomeResolver)) + .putByteArray("SkyLight", column.skyLight()[section]) + .putByteArray("BlockLight", column.blockLight()[section]) + .build(); + } + + /** + * Serialises the given chunk compound without compressing it. + * + * @param data the chunk data to serialise + * @return the uncompressed bytes of the chunk + * @throws IOException if the data cannot be written + */ + public static byte[] serialize(CompoundBinaryTag data) throws IOException { + ByteArrayOutputStream target = new ByteArrayOutputStream(INITIAL_BUFFER); + TAG_WRITER.writeNamed(Map.entry("", data), target, BinaryTagIO.Compression.NONE); + return target.toByteArray(); + } + + /** + * Builds the uncompressed bytes of a chunk in one step. + * + * @param column the chunk to describe + * @param blockResolver the resolver which names the block states + * @param biomeResolver the resolver which names the biomes + * @return the uncompressed bytes of the chunk + * @throws IOException if the data cannot be written + */ + public static byte[] serialize(ChunkColumn column, PaletteEntryResolver blockResolver, PaletteEntryResolver biomeResolver) throws IOException { + return serialize(encode(column, blockResolver, biomeResolver)); + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/support/FakeBlockLightSource.java b/src/jmh/java/net/theevilreaper/aves/benchmark/support/FakeBlockLightSource.java new file mode 100644 index 00000000..6ca74a8b --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/support/FakeBlockLightSource.java @@ -0,0 +1,108 @@ +package net.theevilreaper.aves.benchmark.support; + +import net.theevilreaper.aves.instance.light.BlockFace; +import net.theevilreaper.aves.instance.light.BlockLightSource; +import org.openjdk.jmh.infra.Blackhole; + +/** + * The {@link FakeBlockLightSource} class answers the light properties of a block without touching a + * registry. + *

+ * The real implementation resolves a block through the Minestom registry, which needs a started + * server and would put the cost of that registry into every light measurement. The benchmarks are + * supposed to measure the propagation and the caching of this library, so the source is replaced by + * a table with four known states. + *

+ *

+ * The cost of a single resolution is configurable. {@link net.theevilreaper.aves.instance.light.SectionOpacity} + * caches every distinct state once, and how much that cache is worth depends on how expensive a + * resolution is. + * A source which answers in a nanosecond would make the cache look like pure overhead, which is not + * what a registry lookup behaves like. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +public final class FakeBlockLightSource implements BlockLightSource { + + /** + * The state of a block which neither emits nor blocks light. + */ + public static final int AIR = 0; + + /** + * The state of a block which blocks every face and emits nothing. + */ + public static final int SOLID = 1; + + /** + * The state of a block which blocks every face and emits the highest level. + */ + public static final int GLOWSTONE = 2; + + /** + * The state of a block which blocks only its bottom face, the way a slab does. + */ + public static final int SLAB = 3; + + /** + * The first state id of the filler states which only exist to grow a palette. + * Every filler state behaves like {@link #AIR}. + */ + public static final int FILLER_BASE = 1024; + + private final int resolveCost; + + /** + * Creates a source which resolves a state at the given cost. + * + * @param resolveCost the amount of work tokens a single resolution burns + * @throws IllegalArgumentException if the cost is negative + */ + public FakeBlockLightSource(int resolveCost) { + if (resolveCost < 0) { + throw new IllegalArgumentException("The resolve cost cannot be negative but was " + resolveCost); + } + this.resolveCost = resolveCost; + } + + /** + * Creates a source which resolves a state without any simulated cost. + */ + public FakeBlockLightSource() { + this(0); + } + + /** + * {@inheritDoc} + */ + @Override + public int emission(int stateId) { + burn(); + return stateId == GLOWSTONE ? 15 : 0; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean blocksFace(int stateId, BlockFace face) { + burn(); + return switch (stateId) { + case SOLID, GLOWSTONE -> true; + case SLAB -> face == BlockFace.BOTTOM; + default -> false; + }; + } + + /** + * Burns the configured amount of work so a resolution costs what a registry lookup costs. + */ + private void burn() { + if (this.resolveCost > 0) { + Blackhole.consumeCPU(this.resolveCost); + } + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/support/FakePaletteEntryResolver.java b/src/jmh/java/net/theevilreaper/aves/benchmark/support/FakePaletteEntryResolver.java new file mode 100644 index 00000000..598df39e --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/support/FakePaletteEntryResolver.java @@ -0,0 +1,49 @@ +package net.theevilreaper.aves.benchmark.support; + +import net.kyori.adventure.nbt.CompoundBinaryTag; +import net.theevilreaper.aves.instance.anvil.PaletteEntryResolver; +import org.jetbrains.annotations.Nullable; + +/** + * The {@link FakePaletteEntryResolver} class translates between palette names and ids without a + * registry. + *

+ * The real resolver asks the Minestom registry, which needs a started server. Since the benchmarks + * measure the codec and not the registry, the translation is reduced to a name which carries the id + * in its own text. The shape of the produced entry matches what the format stores, so the amount of + * NBT the codec has to build stays realistic. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +public final class FakePaletteEntryResolver implements PaletteEntryResolver { + + private static final String PREFIX = "aves:state_"; + + /** + * Creates a new resolver. + */ + public FakePaletteEntryResolver() { + } + + /** + * {@inheritDoc} + */ + @Override + public int toId(String name, @Nullable CompoundBinaryTag properties) { + if (!name.startsWith(PREFIX)) { + return 0; + } + return Integer.parseInt(name.substring(PREFIX.length())); + } + + /** + * {@inheritDoc} + */ + @Override + public CompoundBinaryTag toEntry(int id) { + return CompoundBinaryTag.builder().putString("Name", PREFIX + id).build(); + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/support/SectionStates.java b/src/jmh/java/net/theevilreaper/aves/benchmark/support/SectionStates.java new file mode 100644 index 00000000..8bf33f46 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/support/SectionStates.java @@ -0,0 +1,126 @@ +package net.theevilreaper.aves.benchmark.support; + +import java.util.Random; + +/** + * The {@link SectionStates} class generates the block state arrays which the benchmarks feed into + * the codec and into the light engine. + *

+ * Every generator is deterministic. A benchmark which is rerun after a code change has to see the + * exact same input, otherwise the difference between two runs describes the input and not the + * change. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +public final class SectionStates { + + /** + * Blocks the creation of an instance because the class only holds generators. + */ + private SectionStates() { + } + + /** + * Builds a section in which every entry carries the same state. + * This is the shape of the vast majority of the sections of a real world, which are either + * completely air or completely stone. + * + * @param entryCount the amount of entries the section holds + * @param stateId the state every entry carries + * @return the created section + */ + public static int[] uniform(int entryCount, int stateId) { + int[] values = new int[entryCount]; + java.util.Arrays.fill(values, stateId); + return values; + } + + /** + * Builds a section which holds the requested amount of distinct states. + *

+ * The states are not scattered randomly over the whole section. A real section holds runs of + * the same block, so the generator writes the states in contiguous runs whose length follows + * from the amount of distinct states. A purely random fill would produce a palette access + * pattern no world ever shows. + *

+ * + * @param entryCount the amount of entries the section holds + * @param distinctStates the amount of distinct states the section holds + * @param firstStateId the state id the generator starts counting from + * @return the created section + * @throws IllegalArgumentException if the section cannot hold the requested amount of states + */ + public static int[] distinct(int entryCount, int distinctStates, int firstStateId) { + if (distinctStates <= 0 || distinctStates > entryCount) { + throw new IllegalArgumentException( + "A section of " + entryCount + " entries cannot hold " + distinctStates + " distinct states" + ); + } + + if (distinctStates == 1) { + return uniform(entryCount, firstStateId); + } + + int[] values = new int[entryCount]; + Random random = new Random(BenchmarkConstants.SEED); + int runLength = Math.max(entryCount / (distinctStates * 4), 1); + int index = 0; + int state = 0; + + while (index < entryCount) { + int length = Math.min(1 + random.nextInt(runLength * 2), entryCount - index); + java.util.Arrays.fill(values, index, index + length, firstStateId + state); + index += length; + state = (state + 1) % distinctStates; + } + + // The run based fill can miss a state if the section runs out of room, so the tail is + // rewritten to guarantee that every requested state really occurs. + for (int missing = 0; missing < distinctStates; missing++) { + values[entryCount - 1 - missing] = firstStateId + missing; + } + return values; + } + + /** + * Builds a section which mixes air, solid blocks and light emitting blocks. + *

+ * The result is what the light benchmarks run on. The amount of emitting blocks and the share + * of solid blocks are the two properties which decide how much work a propagation performs. + *

+ * + * @param lightSources the amount of light emitting blocks the section holds + * @param occlusionPercent the share of solid blocks in percent + * @return the created section + * @throws IllegalArgumentException if the section cannot hold the requested amount of sources + */ + public static int[] lit(int lightSources, int occlusionPercent) { + int entryCount = BenchmarkConstants.BLOCK_ENTRIES; + + if (lightSources < 0 || lightSources > entryCount) { + throw new IllegalArgumentException( + "A section of " + entryCount + " entries cannot hold " + lightSources + " light sources" + ); + } + + int[] values = new int[entryCount]; + Random random = new Random(BenchmarkConstants.SEED); + + for (int index = 0; index < entryCount; index++) { + values[index] = random.nextInt(100) < occlusionPercent ? FakeBlockLightSource.SOLID : FakeBlockLightSource.AIR; + } + + // The sources are spread evenly instead of being placed randomly. A random placement can + // cluster them, and a cluster performs far less work than the same amount of sources spread + // over the section because the searches of the cluster overlap immediately. + int stride = Math.max(entryCount / Math.max(lightSources, 1), 1); + + for (int source = 0; source < lightSources; source++) { + values[Math.min(source * stride, entryCount - 1)] = FakeBlockLightSource.GLOWSTONE; + } + return values; + } +} diff --git a/src/jmh/java/net/theevilreaper/aves/benchmark/support/package-info.java b/src/jmh/java/net/theevilreaper/aves/benchmark/support/package-info.java new file mode 100644 index 00000000..847fdde0 --- /dev/null +++ b/src/jmh/java/net/theevilreaper/aves/benchmark/support/package-info.java @@ -0,0 +1,12 @@ +/** + * Contains the input generators and the fake collaborators the benchmarks share. + *

+ * Nothing in this package is measured. It exists so every benchmark builds its input in a setup + * method and so no benchmark has to start a Minestom server to resolve a block or a biome. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +package net.theevilreaper.aves.benchmark.support; diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/AnvilChunkException.java b/src/main/java/net/theevilreaper/aves/instance/anvil/AnvilChunkException.java new file mode 100644 index 00000000..d722d149 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/AnvilChunkException.java @@ -0,0 +1,44 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.jetbrains.annotations.ApiStatus; + + +/** + * The {@link AnvilChunkException} is thrown when a chunk which exists on disk cannot be read. + *

+ * The failure has to propagate instead of being reported as an absent chunk. An absent chunk makes + * the server generate a replacement which then overwrites the stored data on the next save, so a + * read failure would silently destroy the very data it failed to read. + *

+ * + *

+ * This type is experimental. The Anvil loader is new and its API may still change while it is + * being validated against real worlds. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public class AnvilChunkException extends RuntimeException { + + /** + * Creates a new exception with the given message. + * + * @param message the message which describes the failure + */ + public AnvilChunkException(String message) { + super(message); + } + + /** + * Creates a new exception with the given message and cause. + * + * @param message the message which describes the failure + * @param cause the failure which caused this one + */ + public AnvilChunkException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnostics.java b/src/main/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnostics.java new file mode 100644 index 00000000..8bed7687 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnostics.java @@ -0,0 +1,201 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; + +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.LongAdder; + +/** + * The {@link AnvilDiagnostics} class throttles repeating warnings of a chunk loader and collects + * the counters which are reported when the loader is closed. + *

+ * A world can hold thousands of chunks which all trigger the same problem, for example a block + * which the server does not know. Logging that problem once per chunk would flood the log without + * adding any information, so every report is reduced to the first occurrence of a distinct name. + * The amount of tracked names is capped because a broken world can contain an unbounded amount of + * unknown names which would otherwise grow the tracking sets without a limit. + *

+ *

+ * Instances are safe to use from multiple threads which is required because a loader reports from + * every thread that loads or saves a chunk. + *

+ * + *

+ * This type is experimental. The Anvil loader is new and its API may still change while it is + * being validated against real worlds. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class AnvilDiagnostics { + + /** + * The highest amount of distinct names a single category tracks. + */ + public static final int MAX_TRACKED_NAMES = 64; + + private final Set unknownBlocks; + private final Set unknownBiomes; + private final AtomicBoolean partialChunkReported; + private final AtomicBoolean sectionRangeReported; + private final LongAdder chunksLoaded; + private final LongAdder chunksSaved; + private final LongAdder errors; + + /** + * Creates a new diagnostics instance with empty counters. + */ + public AnvilDiagnostics() { + this.unknownBlocks = ConcurrentHashMap.newKeySet(); + this.unknownBiomes = ConcurrentHashMap.newKeySet(); + this.partialChunkReported = new AtomicBoolean(); + this.sectionRangeReported = new AtomicBoolean(); + this.chunksLoaded = new LongAdder(); + this.chunksSaved = new LongAdder(); + this.errors = new LongAdder(); + } + + /** + * Checks whether an unknown block name should be reported. + * Only the first occurrence of a name passes so a repeating problem does not flood the log. + * + * @param name the name of the unknown block + * @return true if the caller should log the name, otherwise false + */ + public boolean reportUnknownBlock(String name) { + return track(this.unknownBlocks, name); + } + + /** + * Checks whether an unknown biome name should be reported. + * Only the first occurrence of a name passes so a repeating problem does not flood the log. + * + * @param name the name of the unknown biome + * @return true if the caller should log the name, otherwise false + */ + public boolean reportUnknownBiome(String name) { + return track(this.unknownBiomes, name); + } + + /** + * Checks whether a partially generated chunk should be reported. + * A world which was generated by another server can hold many of them, so the report happens + * only once for the lifetime of the loader. + * + * @return true if the caller should log the problem, otherwise false + */ + public boolean reportPartialChunk() { + return this.partialChunkReported.compareAndSet(false, true); + } + + /** + * Checks whether a section outside of the dimension height should be reported. + * + * @return true if the caller should log the problem, otherwise false + */ + public boolean reportSectionOutOfRange() { + return this.sectionRangeReported.compareAndSet(false, true); + } + + /** + * Counts a chunk which was loaded successfully. + */ + public void countChunkLoaded() { + this.chunksLoaded.increment(); + } + + /** + * Counts a chunk which was saved successfully. + */ + public void countChunkSaved() { + this.chunksSaved.increment(); + } + + /** + * Counts a chunk which could not be loaded or saved. + */ + public void countError() { + this.errors.increment(); + } + + /** + * Returns the amount of chunks which were loaded successfully. + * + * @return the amount of loaded chunks + */ + @Contract(pure = true) + public long chunksLoaded() { + return this.chunksLoaded.sum(); + } + + /** + * Returns the amount of chunks which were saved successfully. + * + * @return the amount of saved chunks + */ + @Contract(pure = true) + public long chunksSaved() { + return this.chunksSaved.sum(); + } + + /** + * Returns the amount of chunks which could not be loaded or saved. + * + * @return the amount of failed chunks + */ + @Contract(pure = true) + public long errors() { + return this.errors.sum(); + } + + /** + * Returns the amount of distinct unknown block names which were reported. + * + * @return the amount of unknown block names + */ + @Contract(pure = true) + public int unknownBlockCount() { + return this.unknownBlocks.size(); + } + + /** + * Returns the amount of distinct unknown biome names which were reported. + * + * @return the amount of unknown biome names + */ + @Contract(pure = true) + public int unknownBiomeCount() { + return this.unknownBiomes.size(); + } + + /** + * Adds the given name to the tracking set as long as the cap is not reached yet. + * + * @param names the set which tracks the names of a category + * @param name the name to track + * @return true if the name was added by this call, otherwise false + */ + private static boolean track(Set names, String name) { + // The size check and the insertion cannot be one atomic step on a set, so racing threads + // could push it past the cap. A slightly relaxed bound is acceptable here because the cap + // exists to protect the heap, not to be an exact quota, but the set is trimmed back so it + // cannot drift upwards over time. + if (names.size() >= MAX_TRACKED_NAMES) { + return false; + } + if (!names.add(name)) { + return false; + } + if (names.size() > MAX_TRACKED_NAMES) { + names.remove(name); + return false; + } + return true; + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java b/src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java new file mode 100644 index 00000000..bc099cc3 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoader.java @@ -0,0 +1,1135 @@ +package net.theevilreaper.aves.instance.anvil; + +import net.kyori.adventure.key.Key; +import net.kyori.adventure.nbt.BinaryTag; +import net.kyori.adventure.nbt.BinaryTagIO; +import net.kyori.adventure.nbt.BinaryTagTypes; +import net.kyori.adventure.nbt.ByteArrayBinaryTag; +import net.kyori.adventure.nbt.CompoundBinaryTag; +import net.kyori.adventure.nbt.ListBinaryTag; +import net.kyori.adventure.nbt.StringBinaryTag; +import net.minestom.server.MinecraftServer; +import net.minestom.server.coordinate.CoordConversion; +import net.minestom.server.instance.Chunk; +import net.minestom.server.instance.ChunkLoader; +import net.minestom.server.instance.Instance; +import net.minestom.server.instance.Section; +import net.minestom.server.instance.block.Block; +import net.minestom.server.instance.palette.Palette; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.Semaphore; + +/** + * The {@link AvesAnvilLoader} class loads and saves chunks in the Anvil format and replaces the + * loader which Minestom ships with. + *

+ * The work of a chunk is split into three stages so the expensive part never happens while a lock + * is held. The chunk state is copied under the read lock of the chunk, the conversion between that + * copy and the compressed bytes runs without any lock at all, and only the transfer of those bytes + * into the region file is guarded. That is the difference which makes parallel access worthwhile, + * because a region file which serializes decompression and parsing gains nothing from more threads. + *

+ *

+ * A chunk which cannot be read is reported as an error instead of being reported as absent. An + * absent chunk makes the server generate a new one which then overwrites the real data on the next + * save, so a read failure has to stay visible. + *

+ *

+ * A region file is never closed while a thread is reading from or writing to it. Every access + * registers itself on the handle first, and a handle which is dropped from the cache while it is + * still registered is closed by the last thread which leaves it. Dropping and closing are therefore + * two separate steps, which is what allows an unload or an eviction to happen at any moment without + * failing the work that is already running. + *

+ *

+ * A loader which was closed refuses further work with an {@link IllegalStateException}. Silently + * ignoring a load would report the chunk as absent and make the server overwrite it, and silently + * ignoring a save would drop chunk data during the shutdown it belongs to. + *

+ * + *

+ * This type is experimental. The Anvil loader is new and its API may still change while it is + * being validated against real worlds. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class AvesAnvilLoader implements ChunkLoader, AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(AvesAnvilLoader.class); + + private static final BinaryTagIO.Reader TAG_READER = BinaryTagIO.unlimitedReader(); + private static final BinaryTagIO.Writer TAG_WRITER = BinaryTagIO.writer(); + + private static final String SECTIONS_KEY = "sections"; + private static final String BLOCK_STATES_KEY = "block_states"; + private static final String BIOMES_KEY = "biomes"; + private static final String BLOCK_ENTITIES_KEY = "block_entities"; + private static final String STATUS_KEY = "Status"; + private static final String LEGACY_STATUS_KEY = "status"; + private static final String FULL_STATUS = "minecraft:full"; + + private static final int BLOCK_ENTRIES = 16 * 16 * 16; + private static final int BIOME_ENTRIES = 4 * 4 * 4; + + /** + * The amount of region files a loader keeps open by default. + */ + public static final int DEFAULT_OPEN_REGION_LIMIT = 64; + + private final int openRegionLimit; + private final int compressionLevel; + private final Path regionDirectory; + private final String dimensionLabel; + private final AnvilDiagnostics diagnostics; + private final PaletteEntryResolver blockResolver; + private final PaletteEntryResolver biomeResolver; + private final Map regions; + private final Map> trackedChunks; + private final Semaphore saveLimit; + private final int dataVersion; + + private volatile boolean closed; + + /** + * Creates a new loader for the given world directory and dimension. + * + * @param worldRoot the root directory of the world + * @param dimension the key of the dimension the loader reads and writes + */ + public AvesAnvilLoader(Path worldRoot, Key dimension) { + this(worldRoot, dimension, DEFAULT_OPEN_REGION_LIMIT); + } + + /** + * Creates a new loader which keeps at most the given amount of region files open. + *

+ * A region file is normally closed as soon as every chunk this loader took from it has been + * unloaded. The limit is the second line of defence for the case that unload calls never + * arrive, for example because chunks stay loaded for the whole lifetime of the server. An + * evicted file is reopened transparently on the next access. + *

+ * + * @param worldRoot the root directory of the world + * @param dimension the key of the dimension the loader reads and writes + * @param openRegionLimit the amount of region files the loader keeps open + * @throws IllegalArgumentException if the limit is not positive + */ + public AvesAnvilLoader(Path worldRoot, Key dimension, int openRegionLimit) { + if (openRegionLimit <= 0) { + throw new IllegalArgumentException("The amount of open region files must be positive but was " + openRegionLimit); + } + this.openRegionLimit = openRegionLimit; + this.compressionLevel = ChunkCompression.DEFAULT_LEVEL; + this.regionDirectory = resolveRegionDirectory(worldRoot, dimension); + this.dimensionLabel = dimension.asString(); + this.diagnostics = new AnvilDiagnostics(); + this.blockResolver = new BlockPaletteResolver(this.diagnostics); + this.biomeResolver = new BiomePaletteResolver(this.diagnostics); + this.regions = new ConcurrentHashMap<>(); + this.trackedChunks = new ConcurrentHashMap<>(); + this.saveLimit = new Semaphore(Math.max(Runtime.getRuntime().availableProcessors(), 2)); + this.dataVersion = MinecraftServer.DATA_VERSION; + + LOGGER.info("Opening the anvil loader for region={} dim={}", this.regionDirectory, this.dimensionLabel); + } + + /** + * Resolves the directory which holds the region files of a dimension. + * A world which still uses the layout without a dimension directory keeps working because the + * legacy directory is used when it exists and the current one does not. + * + * @param worldRoot the root directory of the world + * @param dimension the key of the dimension + * @return the directory which holds the region files + */ + @Contract(pure = true) + private static Path resolveRegionDirectory(Path worldRoot, Key dimension) { + Path current = worldRoot.resolve("dimensions").resolve(dimension.namespace()).resolve(dimension.value()).resolve("region"); + Path legacy = worldRoot.resolve("region"); + + if (!Files.isDirectory(current) && Files.isDirectory(legacy)) { + return legacy; + } + return current; + } + + /** + * {@inheritDoc} + *

+ * The region file is registered as in use for the duration of the read, so an unload or an + * eviction which happens in parallel cannot close the file this call is reading from. + *

+ * + * @throws IllegalStateException if the loader was already closed or is closed while the call + * is looking for the region file + */ + @Override + public @Nullable Chunk loadChunk(Instance instance, int chunkX, int chunkZ) { + ensureOpen(); + RegionHandle handle; + + // The acquisition stays outside of the block which reports a failed chunk. It refuses its + // work once the loader is closed, and a shutdown which arrives during a load is a lifecycle + // event of the caller rather than a chunk which could not be read. + try { + handle = acquireRegion(chunkX, chunkZ, false); + } catch (IOException exception) { + throw failedLoad(chunkX, chunkZ, exception); + } + + if (handle == null) { + return null; + } + + try { + RegionFile.RawChunk raw; + + try { + raw = handle.file().readRaw(chunkX, chunkZ); + } finally { + releaseRegion(handle); + } + + if (raw == null) { + return null; + } + + CompoundBinaryTag data = TAG_READER.read(new ByteArrayInputStream(raw.decompress()), BinaryTagIO.Compression.NONE); + + if (!isFullyGenerated(data)) { + if (this.diagnostics.reportPartialChunk()) { + LOGGER.warn( + "Skipping a chunk which is not fully generated chunk=[{},{}] region={} dim={}", + chunkX, chunkZ, this.regionDirectory, this.dimensionLabel + ); + } + return null; + } + + Chunk chunk = instance.getChunkSupplier().createChunk(instance, chunkX, chunkZ); + // The conversion runs before the lock is taken so only the transfer into the chunk is + // guarded. That is what keeps parallel loading worthwhile. + List sections = decodeSections(chunk, data); + + chunk.lockWriteLock(); + try { + for (DecodedSection section : sections) { + section.applyTo(chunk); + } + applyBlockEntities(chunk, data); + } finally { + chunk.unlockWriteLock(); + } + trackChunk(chunkX, chunkZ); + this.diagnostics.countChunkLoaded(); + return chunk; + } catch (IOException | RuntimeException exception) { + throw failedLoad(chunkX, chunkZ, exception); + } + } + + /** + * Reports a chunk which could not be read and builds the exception which carries that failure + * to the caller. + *

+ * Reporting the chunk as absent would make the server generate a replacement which overwrites + * the real data on the next save, so the failure has to propagate. + *

+ * + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @param exception the failure which stopped the load + * @return the exception the caller has to throw + */ + private AnvilChunkException failedLoad(int chunkX, int chunkZ, Throwable exception) { + this.diagnostics.countError(); + LOGGER.error( + "Failed to load the chunk chunk=[{},{}] region={} dim={}", + chunkX, chunkZ, this.regionDirectory, this.dimensionLabel, exception + ); + MinecraftServer.getExceptionManager().handleException(exception); + return new AnvilChunkException("The chunk " + chunkX + "/" + chunkZ + " could not be loaded", exception); + } + + /** + * {@inheritDoc} + * + * @throws IllegalStateException if the loader was already closed or is closed while the call + * is looking for the region file + */ + @Override + public void saveChunk(Chunk chunk) { + ensureOpen(); + int chunkX = chunk.getChunkX(); + int chunkZ = chunk.getChunkZ(); + + try { + CompoundBinaryTag data = snapshot(chunk); + ByteArrayOutputStream target = new ByteArrayOutputStream(64 * 1024); + TAG_WRITER.writeNamed(Map.entry("", data), target, BinaryTagIO.Compression.NONE); + + writeToRegion(chunkX, chunkZ, ChunkCompression.ZLIB.compress(target.toByteArray(), this.compressionLevel)); + this.diagnostics.countChunkSaved(); + } catch (IllegalStateException exception) { + // The loader was closed while this save was running. Counting that as a failed chunk + // would hide the reason behind a data error, so the refusal reaches the caller as it is. + throw exception; + } catch (IOException | RuntimeException exception) { + this.diagnostics.countError(); + LOGGER.error( + "Failed to save the chunk chunk=[{},{}] region={} dim={}", + chunkX, chunkZ, this.regionDirectory, this.dimensionLabel, exception + ); + MinecraftServer.getExceptionManager().handleException(exception); + } + } + + /** + * {@inheritDoc} + *

+ * The chunks are grouped by their region file and every group is handled by a single task. The + * default implementation starts one thread per chunk which lets thousands of them compete for + * the same region locks while every chunk snapshot is held in memory at the same time. + *

+ * + * @throws IllegalStateException if the loader was already closed + */ + @Override + public void saveChunks(Collection chunks) { + ensureOpen(); + Map> grouped = new HashMap<>(); + + for (Chunk chunk : chunks) { + long region = CoordConversion.regionIndex( + RegionConstants.chunkToRegion(chunk.getChunkX()), + RegionConstants.chunkToRegion(chunk.getChunkZ()) + ); + grouped.computeIfAbsent(region, ignored -> new ArrayList<>()).add(chunk); + } + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(grouped.size()); + + for (List group : grouped.values()) { + futures.add(executor.submit(() -> { + this.saveLimit.acquire(); + try { + for (Chunk chunk : group) { + saveChunk(chunk); + } + } finally { + this.saveLimit.release(); + } + return null; + })); + } + awaitAll(futures); + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean supportsParallelLoading() { + return true; + } + + /** + * {@inheritDoc} + */ + @Override + public boolean supportsParallelSaving() { + return true; + } + + /** + * {@inheritDoc} + *

+ * The region file of the chunk is closed once every chunk this loader took from it has been + * unloaded. Only chunks this loader handled itself are tracked, because a loader also receives + * unload calls for chunks it never loaded. An unload call for such a chunk is ignored instead + * of closing a file which is still in use. + *

+ */ + @Override + public void unloadChunk(Chunk chunk) { + int chunkX = chunk.getChunkX(); + int chunkZ = chunk.getChunkZ(); + long index = regionIndex(chunkX, chunkZ); + Set chunks = this.trackedChunks.get(index); + + if (chunks == null || !chunks.remove(CoordConversion.chunkIndex(chunkX, chunkZ))) { + return; + } + + LOGGER.trace("Unloading the chunk chunk=[{},{}] dim={}", chunkX, chunkZ, this.dimensionLabel); + + if (!chunks.isEmpty()) { + return; + } + // The removal has to be conditional, another thread may have registered a chunk since the + // emptiness check above. + if (this.trackedChunks.remove(index, chunks)) { + closeRegion(index); + } + } + + /** + * Drops the region file with the given index from the cache and closes it. + *

+ * A file which is still being read from or written to is only dropped here. The thread which + * leaves it last performs the actual close, so an unload cannot break a load which is already + * running. + *

+ * + * @param index the index of the region file + */ + private void closeRegion(long index) { + RegionHandle handle = this.regions.remove(index); + + if (handle != null) { + retire(handle, "after its last chunk was unloaded"); + } + } + + /** + * Closes the given handle unless a thread is still using it. + * The handle has to be removed from the cache before this method is called, otherwise a thread + * could register itself on a file which is about to be closed. + * + * @param handle the handle which was dropped from the cache + * @param reason the reason which is written into the log line of a successful close + */ + private void retire(RegionHandle handle, String reason) { + if (handle.retire()) { + closeQuietly(handle, reason); + } + } + + /** + * Closes the file of the given handle and reports a failure instead of propagating it. + * + * @param handle the handle whose file is closed + * @param reason the reason which is written into the log line of a successful close + */ + private void closeQuietly(RegionHandle handle, String reason) { + try { + handle.file().flush(); + handle.file().close(); + LOGGER.debug("Closed the region file region={} dim={} {}", handle.file().path(), this.dimensionLabel, reason); + } catch (IOException exception) { + this.diagnostics.countError(); + LOGGER.error("Failed to close the region file region={} dim={}", handle.file().path(), this.dimensionLabel, exception); + } + } + + /** + * Releases a handle which was obtained by {@link #acquireRegion(int, int, boolean)}. + * The file is closed here when this thread was the last user of a handle which had already been + * dropped from the cache. + * + * @param handle the handle to release + */ + private void releaseRegion(RegionHandle handle) { + if (handle.release()) { + closeQuietly(handle, "after its last user finished"); + } + } + + /** + * Verifies that the loader is still usable. + * + * @throws IllegalStateException if the loader was already closed + */ + private void ensureOpen() { + if (this.closed) { + throw new IllegalStateException( + "The anvil loader for region=" + this.regionDirectory + " dim=" + this.dimensionLabel + " is closed" + ); + } + } + + /** + * Records that this loader handled the given chunk so its region file can be released once + * every chunk of that file has been unloaded again. + * + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + */ + private void trackChunk(int chunkX, int chunkZ) { + this.trackedChunks + .computeIfAbsent(regionIndex(chunkX, chunkZ), ignored -> ConcurrentHashMap.newKeySet()) + .add(CoordConversion.chunkIndex(chunkX, chunkZ)); + } + + /** + * Calculates the index of the region file which holds the given chunk. + * + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @return the index of the region file + */ + @Contract(pure = true) + private static long regionIndex(int chunkX, int chunkZ) { + return CoordConversion.regionIndex(RegionConstants.chunkToRegion(chunkX), RegionConstants.chunkToRegion(chunkZ)); + } + + /** + * Returns the diagnostics which collect the counters of this loader. + * + * @return the diagnostics of the loader + */ + @Contract(pure = true) + public AnvilDiagnostics diagnostics() { + return this.diagnostics; + } + + /** + * Closes every region file the loader opened and reports a summary of its work. + *

+ * A loader is closed while the tasks of the server are still running, because the loader reports + * parallel work as supported and therefore receives one task per chunk. A file which such a task + * is still using is dropped from the cache here and closed by that task when it finishes, so no + * handle survives the shutdown. Every later call is rejected, which is what stops a task from + * opening a file that nobody would close again. + *

+ * + * @throws IOException if a region file cannot be closed + */ + @Override + public synchronized void close() throws IOException { + if (this.closed) { + return; + } + // The flag is raised before the cache is emptied. A thread which publishes a handle reads + // the flag after publishing it, so either that thread sees the flag or the loop below sees + // the handle, and the file is closed in both cases. + this.closed = true; + IOException failure = null; + + for (Long index : List.copyOf(this.regions.keySet())) { + RegionHandle handle = this.regions.remove(index); + + if (handle == null) { + continue; + } + if (!handle.retire()) { + LOGGER.debug("Leaving the region file region={} dim={} to the task which is still using it", + handle.file().path(), this.dimensionLabel); + continue; + } + + try { + handle.file().flush(); + handle.file().close(); + } catch (IOException exception) { + failure = exception; + LOGGER.error("Failed to close the region file region={} dim={}", handle.file().path(), this.dimensionLabel, exception); + } + } + this.regions.clear(); + this.trackedChunks.clear(); + logSummary(); + + if (failure != null) { + throw failure; + } + } + + /** + * Writes the summary of the loader. The line reports on the error level when at least one + * chunk failed so a shutdown which lost data does not look like a clean one. + */ + private void logSummary() { + long errors = this.diagnostics.errors(); + String message = "Closing the anvil loader after {} loaded and {} saved chunks with {} errors," + + " {} unknown blocks and {} unknown biomes region={} dim={}"; + + if (errors > 0) { + LOGGER.warn( + message, this.diagnostics.chunksLoaded(), this.diagnostics.chunksSaved(), errors, + this.diagnostics.unknownBlockCount(), this.diagnostics.unknownBiomeCount(), + this.regionDirectory, this.dimensionLabel + ); + return; + } + LOGGER.info( + message, this.diagnostics.chunksLoaded(), this.diagnostics.chunksSaved(), errors, + this.diagnostics.unknownBlockCount(), this.diagnostics.unknownBiomeCount(), + this.regionDirectory, this.dimensionLabel + ); + } + + /** + * Writes the given payload into the region file of the chunk. + *

+ * The handle is registered as in use for the duration of the write, so an eviction which + * happens in parallel drops the file from the cache without closing it under this thread. The + * write therefore needs no retry. + *

+ * + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @param payload the compressed payload of the chunk + * @throws IOException if the chunk cannot be written + */ + private void writeToRegion(int chunkX, int chunkZ, byte[] payload) throws IOException { + RegionHandle handle = acquireRegion(chunkX, chunkZ, true); + + if (handle == null) { + throw new IOException("The region file for the chunk " + chunkX + "/" + chunkZ + " could not be created"); + } + + try { + handle.file().writeRaw(chunkX, chunkZ, ChunkCompression.ZLIB, payload); + } finally { + releaseRegion(handle); + } + } + + /** + * Returns a registered handle for the region file which holds the given chunk. + *

+ * The registration is what keeps the file open for the caller. A handle which was dropped from + * the cache in the meantime cannot be registered any more, and the loop then opens the file + * again instead of handing back a file which is about to be closed. Every returned handle has + * to be released through {@link #releaseRegion(RegionHandle)}. + *

+ *

+ * The file is opened outside of the mapping function of the map because opening it performs + * blocking work. Doing that inside the mapping function blocks a bin of the map for the whole + * duration and has already caused a deadlock in the loader of Minestom. + *

+ * + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @param create whether the file should be created when it does not exist yet + * @return the registered handle or null if the file does not exist and should not be created + * @throws IOException if the file cannot be opened + * @throws IllegalStateException if the loader was closed while the file was being opened + */ + private @Nullable RegionHandle acquireRegion(int chunkX, int chunkZ, boolean create) throws IOException { + int regionX = RegionConstants.chunkToRegion(chunkX); + int regionZ = RegionConstants.chunkToRegion(chunkZ); + long index = CoordConversion.regionIndex(regionX, regionZ); + Path path = this.regionDirectory.resolve("r." + regionX + "." + regionZ + ".mca"); + + while (true) { + RegionHandle cached = this.regions.get(index); + + if (cached != null) { + if (cached.acquire()) { + return cached; + } + // The handle was dropped between the lookup and the registration, so it is on its + // way out and a new one has to be opened. + continue; + } + + if (!create && !Files.exists(path)) { + return null; + } + ensureOpen(); + + RegionHandle opened = new RegionHandle(RegionFile.open(path)); + RegionHandle previous = this.regions.putIfAbsent(index, opened); + + if (previous != null) { + opened.file().close(); + continue; + } + // The loader can be closed between the check above and this publication. The flag is + // read after publishing, so whoever of the two threads loses the race still sees the + // work of the other one and the file cannot survive as an unclosed handle. + if (this.closed) { + if (this.regions.remove(index, opened)) { + retire(opened, "because the loader was closed while it was being opened"); + } + ensureOpen(); + } + + LOGGER.debug("Opened the region file region={} dim={}", path, this.dimensionLabel); + + if (!opened.acquire()) { + continue; + } + evictRegions(index); + return opened; + } + } + + /** + * Drops region files from the cache until the configured limit is met again. + *

+ * The file which was just opened is never dropped so the caller keeps a cached handle. A file + * which another thread is still using is only dropped here; that thread closes it when it + * finishes, which can keep the loader above its limit for the duration of a single access. + *

+ * + * @param keep the index of the region file which must stay cached + */ + private void evictRegions(long keep) { + for (Map.Entry entry : this.regions.entrySet()) { + if (this.regions.size() <= this.openRegionLimit) { + return; + } + if (entry.getKey() == keep || !this.regions.remove(entry.getKey(), entry.getValue())) { + continue; + } + retire(entry.getValue(), "to stay below the open file limit"); + } + } + + /** + * The {@link RegionHandle} class ties a region file to the amount of threads which are working + * with it, which is what allows the file to be dropped from the cache at any moment without + * closing it under a thread that is still reading or writing. + *

+ * A handle is either usable, which means it can accept further users, or retired, which means it + * was dropped from the cache and is closed as soon as its last user leaves. A retired handle + * never becomes usable again, so a thread which finds one has to open the file anew. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ + private static final class RegionHandle { + + private final RegionFile file; + + private int users; + private boolean retired; + + /** + * Creates a handle for the given region file. + * + * @param file the region file the handle guards + */ + private RegionHandle(RegionFile file) { + this.file = file; + } + + /** + * Returns the region file of this handle. + * + * @return the guarded region file + */ + @Contract(pure = true) + private RegionFile file() { + return this.file; + } + + /** + * Registers the calling thread as a user of the region file. + * + * @return true if the file may be used, false if the handle is already retired + */ + private synchronized boolean acquire() { + if (this.retired) { + return false; + } + this.users++; + return true; + } + + /** + * Removes the calling thread from the users of the region file. + * + * @return true if the caller has to close the file because it was the last user of a + * retired handle, otherwise false + */ + private synchronized boolean release() { + this.users--; + return this.retired && this.users == 0; + } + + /** + * Marks the handle as dropped from the cache so it cannot accept further users. + * + * @return true if the caller has to close the file because no thread is using it, false if + * the last user closes it instead + */ + private synchronized boolean retire() { + this.retired = true; + return this.users == 0; + } + } + + /** + * Returns the amount of region files the loader currently keeps open. + * + * @return the amount of open region files + */ + @Contract(pure = true) + public int openRegionCount() { + return this.regions.size(); + } + + /** + * Checks whether the given chunk data describes a fully generated chunk. + * The key is read in both spellings because Minestom writes it in lower case while the game + * itself writes it capitalised. + * + * @param data the chunk data to check + * @return true if the chunk is fully generated, otherwise false + */ + @Contract(pure = true) + private static boolean isFullyGenerated(CompoundBinaryTag data) { + String status = NbtReads.optionalString(data, STATUS_KEY); + + if (status == null) { + status = NbtReads.optionalString(data, LEGACY_STATUS_KEY); + } + return status == null || FULL_STATUS.equals(status); + } + + /** + * Converts the sections of the given chunk data without touching the chunk. + * The result is applied to the chunk afterwards while the write lock is held, which keeps the + * expensive conversion out of the guarded section. + * + * @param chunk the chunk the sections belong to + * @param data the chunk data to read + * @return the converted sections + * @throws IOException if a section is malformed + */ + private List decodeSections(Chunk chunk, CompoundBinaryTag data) throws IOException { + ListBinaryTag sections = NbtReads.optionalList(data, SECTIONS_KEY, BinaryTagTypes.COMPOUND); + List decoded = new ArrayList<>(sections.size()); + + for (int index = 0; index < sections.size(); index++) { + CompoundBinaryTag sectionData = sections.getCompound(index); + int sectionY = NbtReads.integer(sectionData, "Y"); + + if (sectionY < chunk.getMinSection() || sectionY >= chunk.getMaxSection()) { + // The game stores one section below and one above the world for lighting purposes. + LOGGER.trace("Skipping the section {} outside of the world chunk=[{},{}]", sectionY, chunk.getChunkX(), chunk.getChunkZ()); + continue; + } + + CompoundBinaryTag blockStates = NbtReads.optionalCompound(sectionData, BLOCK_STATES_KEY); + CompoundBinaryTag biomes = NbtReads.optionalCompound(sectionData, BIOMES_KEY); + + decoded.add(new DecodedSection( + sectionY, + blockStates == null ? null : SectionCodec.decode(blockStates, this.blockResolver, BLOCK_ENTRIES, Palette.BLOCK_PALETTE_MIN_BITS), + biomes == null ? null : SectionCodec.decodeBiomes(biomes, this.biomeResolver, BIOME_ENTRIES, Palette.BIOME_PALETTE_MIN_BITS), + lightArray(sectionData, "SkyLight"), + lightArray(sectionData, "BlockLight") + )); + } + return decoded; + } + + /** + * Reads a light array of a section. + * + * @param sectionData the section data to read + * @param key the key of the light array + * @return the light array or null if the section carries none + */ + @Contract(pure = true) + private static byte @Nullable [] lightArray(CompoundBinaryTag sectionData, String key) { + if (sectionData.get(key) instanceof ByteArrayBinaryTag light && light.size() == 2048) { + return light.value(); + } + return null; + } + + /** + * The {@link DecodedSection} record holds the converted content of a single section until it is + * transferred into a chunk under its write lock. + * + * @param sectionY the vertical index of the section + * @param blocks the converted block palette or null if the section carries none + * @param biomes the converted biome palette or null if the section carries none + * @param skyLight the stored sky light or null if the section carries none + * @param blockLight the stored block light or null if the section carries none + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ + private record DecodedSection( + int sectionY, + @Nullable PaletteData blocks, + @Nullable PaletteData biomes, + byte @Nullable [] skyLight, + byte @Nullable [] blockLight + ) { + + /** + * Transfers the content of this section into the given chunk. + * The caller has to hold the write lock of the chunk. + * + * @param chunk the chunk which receives the content + * @throws IOException if a palette holds an index outside of its palette + */ + private void applyTo(Chunk chunk) throws IOException { + Section section = chunk.getSection(this.sectionY); + + if (this.skyLight != null) { + section.skyLight().set(this.skyLight); + } + if (this.blockLight != null) { + section.blockLight().set(this.blockLight); + } + if (this.blocks != null) { + apply(section.blockPalette(), this.blocks); + } + if (this.biomes != null) { + apply(section.biomePalette(), this.biomes); + } + } + } + + /** + * Transfers the given palette representation into a palette of Minestom. + * + * @param palette the palette which receives the values + * @param data the palette representation to transfer + * @throws IOException if the representation holds an index outside of its palette + */ + private static void apply(Palette palette, PaletteData data) throws IOException { + if (data.isSingleValue()) { + palette.fill(data.singleValue()); + return; + } + long[] packed = data.packed(); + + if (packed != null && data.bitsPerEntry() == BitPacker.bitsPerEntry(data.palette().length, palette.bitsPerEntry())) { + palette.load(data.palette(), packed); + return; + } + + int[] values = data.unpack(); + palette.setAll((x, y, z) -> values[index(x, y, z, palette.dimension())]); + } + + /** + * Calculates the index of a coordinate inside a palette of the given dimension. + * + * @param x the x coordinate inside the palette + * @param y the y coordinate inside the palette + * @param z the z coordinate inside the palette + * @param dimension the edge length of the palette + * @return the index of the coordinate + */ + @Contract(pure = true) + private static int index(int x, int y, int z, int dimension) { + return (y * dimension + z) * dimension + x; + } + + /** + * Applies the stored block entities to the chunk. + * + * @param chunk the chunk which receives the block entities + * @param data the chunk data to read + * @throws IOException if a block entity is malformed + */ + private void applyBlockEntities(Chunk chunk, CompoundBinaryTag data) throws IOException { + ListBinaryTag entities = NbtReads.optionalList(data, BLOCK_ENTITIES_KEY, BinaryTagTypes.COMPOUND); + + for (int index = 0; index < entities.size(); index++) { + CompoundBinaryTag entity = entities.getCompound(index); + // The stored position is a world coordinate and has to be mapped back into the chunk. + int x = NbtReads.integer(entity, "x") & (Chunk.CHUNK_SIZE_X - 1); + int y = NbtReads.integer(entity, "y"); + int z = NbtReads.integer(entity, "z") & (Chunk.CHUNK_SIZE_Z - 1); + + Block block = chunk.getBlock(x, y, z); + CompoundBinaryTag.Builder tags = CompoundBinaryTag.builder(); + + for (Map.Entry entry : entity) { + String key = entry.getKey(); + + if (!"x".equals(key) && !"y".equals(key) && !"z".equals(key) && !"id".equals(key) && !"keepPacked".equals(key)) { + tags.put(key, entry.getValue()); + } + } + + // The id names the block handler. Without resolving it the handler of every block + // entity would be lost even though it is written back on the next save. + if (entity.get("id") instanceof StringBinaryTag id) { + block = block.withHandler(MinecraftServer.getBlockManager().getHandlerOrDummy(id.value())); + } + + CompoundBinaryTag nbt = tags.build(); + chunk.setBlock(x, y, z, nbt.size() == 0 ? block : block.withNbt(nbt)); + } + } + + /** + * Builds the chunk data of the given chunk. + * The state is copied under the read lock and everything else happens without it so the chunk + * stays usable while its data is converted. + * + * @param chunk the chunk to describe + * @return the chunk data of the chunk + * @throws IOException if the chunk data cannot be built + */ + private CompoundBinaryTag snapshot(Chunk chunk) throws IOException { + List
copies; + List blockEntities = new ArrayList<>(); + + chunk.lockReadLock(); + try { + List
sections = chunk.getSections(); + copies = new ArrayList<>(sections.size()); + + for (Section section : sections) { + copies.add(section.clone()); + } + collectBlockEntities(chunk, blockEntities); + } finally { + chunk.unlockReadLock(); + } + + ListBinaryTag.Builder sections = ListBinaryTag.builder(BinaryTagTypes.COMPOUND); + + for (int index = 0; index < copies.size(); index++) { + sections.add(encodeSection(copies.get(index), chunk.getMinSection() + index)); + } + + ListBinaryTag.Builder entities = ListBinaryTag.builder(BinaryTagTypes.COMPOUND); + blockEntities.forEach(entities::add); + + return CompoundBinaryTag.builder() + .putInt("DataVersion", this.dataVersion) + .putInt("xPos", chunk.getChunkX()) + .putInt("zPos", chunk.getChunkZ()) + .putInt("yPos", chunk.getMinSection()) + .putString(STATUS_KEY, FULL_STATUS) + .putLong("LastUpdate", 0L) + .put(SECTIONS_KEY, sections.build()) + .put(BLOCK_ENTITIES_KEY, entities.build()) + .build(); + } + + /** + * Collects the block entities of the given chunk. + * Every block which carries data or a handler becomes a block entity, including the blocks of a + * section which holds a single value. The loader of Minestom misses those. + * + * @param chunk the chunk to read + * @param target the list which receives the block entities + */ + private static void collectBlockEntities(Chunk chunk, List target) { + int minY = chunk.getMinSection() * Chunk.CHUNK_SECTION_SIZE; + int maxY = chunk.getMaxSection() * Chunk.CHUNK_SECTION_SIZE; + + for (int y = minY; y < maxY; y++) { + for (int x = 0; x < Chunk.CHUNK_SIZE_X; x++) { + for (int z = 0; z < Chunk.CHUNK_SIZE_Z; z++) { + Block block = chunk.getBlock(x, y, z, Block.Getter.Condition.CACHED); + + if (block == null) { + continue; + } + + CompoundBinaryTag nbt = block.nbt(); + boolean hasHandler = block.handler() != null; + + if (nbt == null && !hasHandler) { + continue; + } + + CompoundBinaryTag.Builder entity = CompoundBinaryTag.builder(); + + if (nbt != null) { + for (Map.Entry entry : nbt) { + entity.put(entry.getKey(), entry.getValue()); + } + } + if (hasHandler) { + entity.putString("id", block.handler().getKey().asString()); + } + // The format stores the position in world coordinates, not in chunk local ones. + target.add(entity + .putInt("x", chunk.getChunkX() * Chunk.CHUNK_SIZE_X + x) + .putInt("y", y) + .putInt("z", chunk.getChunkZ() * Chunk.CHUNK_SIZE_Z + z) + .build()); + } + } + } + } + + /** + * Builds the data of a single section. + * + * @param section the section to describe + * @param sectionY the vertical index of the section + * @return the data of the section + */ + private CompoundBinaryTag encodeSection(Section section, int sectionY) { + return CompoundBinaryTag.builder() + .putByte("Y", (byte) sectionY) + .put(BLOCK_STATES_KEY, SectionCodec.encode(read(section.blockPalette(), BLOCK_ENTRIES, Palette.BLOCK_PALETTE_MIN_BITS), this.blockResolver)) + .put(BIOMES_KEY, SectionCodec.encodeBiomes(read(section.biomePalette(), BIOME_ENTRIES, Palette.BIOME_PALETTE_MIN_BITS), this.biomeResolver)) + .putByteArray("SkyLight", section.skyLight().array()) + .putByteArray("BlockLight", section.blockLight().array()) + .build(); + } + + /** + * Reads the values of a palette of Minestom into the representation of the codec. + * + * @param palette the palette to read + * @param entryCount the amount of entries the palette holds + * @param minBitsPerEntry the smallest amount of bits the palette type allows + * @return the palette representation of the palette + */ + private static PaletteData read(Palette palette, int entryCount, int minBitsPerEntry) { + int[] values = new int[entryCount]; + int dimension = palette.dimension(); + palette.getAll((x, y, z, value) -> values[index(x, y, z, dimension)] = value); + return PaletteData.encode(values, minBitsPerEntry); + } + + /** + * Waits for every given task and reports the failures of them. + * + * @param futures the tasks to wait for + */ + private void awaitAll(List> futures) { + for (Future future : futures) { + try { + future.get(); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + return; + } catch (ExecutionException exception) { + this.diagnostics.countError(); + LOGGER.error("Failed to save a group of chunks region={} dim={}", this.regionDirectory, this.dimensionLabel, exception.getCause()); + MinecraftServer.getExceptionManager().handleException(exception.getCause()); + } + } + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/BiomePaletteResolver.java b/src/main/java/net/theevilreaper/aves/instance/anvil/BiomePaletteResolver.java new file mode 100644 index 00000000..69e33472 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/BiomePaletteResolver.java @@ -0,0 +1,135 @@ +package net.theevilreaper.aves.instance.anvil; + +import net.kyori.adventure.nbt.CompoundBinaryTag; +import net.minestom.server.MinecraftServer; +import net.minestom.server.registry.DynamicRegistry; +import net.minestom.server.registry.RegistryKey; +import net.minestom.server.world.biome.Biome; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.function.Supplier; + +/** + * The {@link BiomePaletteResolver} class translates between the biome entries of the Anvil format + * and the biome ids of the server registry. + *

+ * A biome the registry does not know is replaced with the plains biome instead of failing, which + * follows the behaviour of the built-in loader. Every replaced name is reported once through the + * diagnostics. + *

+ *

+ * The registry is resolved on the first use instead of in a static initializer or in the + * constructor. Both would require a running server before the class is touched for the first time, + * which prevents a loader from being created while the server is still starting. + *

+ * + *

+ * This type is experimental. The Anvil loader is new and its API may still change while it is + * being validated against real worlds. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class BiomePaletteResolver implements PaletteEntryResolver { + + private static final Logger LOGGER = LoggerFactory.getLogger(BiomePaletteResolver.class); + + private static final String NAME_KEY = "Name"; + + private final AnvilDiagnostics diagnostics; + private final Supplier> registrySupplier; + + private volatile @Nullable Registries resolved; + + /** + * Creates a new resolver which uses the biome registry of the running server. + * + * @param diagnostics the diagnostics which throttle the reports + */ + public BiomePaletteResolver(AnvilDiagnostics diagnostics) { + this(diagnostics, MinecraftServer::getBiomeRegistry); + } + + /** + * Creates a new resolver which uses the registry the given supplier provides. + *

+ * The registry is resolved on the first use instead of in the constructor. A loader is often + * created while the server is still starting and reading the registry too early would fail + * before the loader ever touches a chunk. + *

+ * + * @param diagnostics the diagnostics which throttle the reports + * @param registrySupplier the supplier which provides the registry of the known biomes + */ + public BiomePaletteResolver(AnvilDiagnostics diagnostics, Supplier> registrySupplier) { + this.diagnostics = diagnostics; + this.registrySupplier = registrySupplier; + } + + /** + * Returns the registry of this resolver and resolves it on the first call. + * + * @return the registry and the id of the fallback biome + */ + private Registries registries() { + Registries current = this.resolved; + + if (current != null) { + return current; + } + + synchronized (this) { + if (this.resolved == null) { + DynamicRegistry registry = this.registrySupplier.get(); + this.resolved = new Registries(registry, registry.getId(Biome.PLAINS)); + } + return this.resolved; + } + } + + /** + * The {@link Registries} record holds the resolved registry together with the id of the biome + * which replaces an unknown one. + * + * @param registry the registry which holds the known biomes + * @param fallbackId the id of the biome which replaces an unknown one + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ + private record Registries(DynamicRegistry registry, int fallbackId) { + } + + /** + * {@inheritDoc} + */ + @Override + public int toId(String name, @Nullable CompoundBinaryTag properties) { + Registries registries = registries(); + int id = registries.registry().getId(RegistryKey.unsafeOf(name)); + + if (id != -1) { + return id; + } + if (this.diagnostics.reportUnknownBiome(name)) { + LOGGER.warn("The biome '{}' is unknown and is replaced with plains, further chunks with it are not reported", name); + } + return registries.fallbackId(); + } + + /** + * {@inheritDoc} + */ + @Override + public CompoundBinaryTag toEntry(int id) { + RegistryKey key = registries().registry().getKey(id); + String name = key == null ? Biome.PLAINS.key().asString() : key.key().asString(); + return CompoundBinaryTag.builder().putString(NAME_KEY, name).build(); + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/BitPacker.java b/src/main/java/net/theevilreaper/aves/instance/anvil/BitPacker.java new file mode 100644 index 00000000..497487b8 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/BitPacker.java @@ -0,0 +1,167 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; + +/** + * The {@link BitPacker} converts palette indices into the packed long array representation + * which the Anvil format uses for block and biome data. + *

+ * Since the world format of Minecraft 1.16 an entry never spans two longs. Every long stores + * {@code 64 / bitsPerEntry} entries and the remaining upper bits stay empty. The class only + * contains pure functions so the encoding can be verified without any file or server access. + *

+ * + *

+ * This type is experimental. The Anvil loader is new and its API may still change while it is + * being validated against real worlds. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class BitPacker { + + private static final int BITS_PER_LONG = Long.SIZE; + + private BitPacker() { + } + + /** + * Calculates the amount of bits which are required to address every entry of a palette. + * The result never falls below the given minimum which the format defines per palette type. + * + * @param paletteSize the amount of entries the palette holds + * @param minBitsPerEntry the smallest amount of bits the palette type allows + * @return the amount of bits per entry + * @throws IllegalArgumentException if the palette does not hold at least one entry + */ + @Contract(pure = true) + public static int bitsPerEntry(int paletteSize, int minBitsPerEntry) { + if (paletteSize <= 0) { + throw new IllegalArgumentException("The palette must hold at least one entry but held " + paletteSize); + } + int required = Integer.SIZE - Integer.numberOfLeadingZeros(paletteSize - 1); + return Math.max(Math.max(required, 1), minBitsPerEntry); + } + + /** + * Calculates the amount of longs which are required to store the given amount of entries. + * + * @param entryCount the amount of entries to store + * @param bitsPerEntry the amount of bits a single entry occupies + * @return the amount of longs which are required + * @throws IllegalArgumentException if the amount of bits per entry is not usable + */ + @Contract(pure = true) + public static int expectedLongCount(int entryCount, int bitsPerEntry) { + int entriesPerLong = entriesPerLong(bitsPerEntry); + return (entryCount + entriesPerLong - 1) / entriesPerLong; + } + + /** + * Packs the given entries into a long array without letting an entry span two longs. + * + * @param values the entries to pack + * @param bitsPerEntry the amount of bits a single entry occupies + * @return the packed representation of the given entries + * @throws IllegalArgumentException if the amount of bits per entry is not usable + */ + @Contract(pure = true) + public static long[] pack(int[] values, int bitsPerEntry) { + int entriesPerLong = entriesPerLong(bitsPerEntry); + long mask = mask(bitsPerEntry); + long[] packed = new long[expectedLongCount(values.length, bitsPerEntry)]; + + for (int index = 0; index < values.length; index++) { + int longIndex = index / entriesPerLong; + int bitOffset = (index % entriesPerLong) * bitsPerEntry; + packed[longIndex] |= (values[index] & mask) << bitOffset; + } + return packed; + } + + /** + * Unpacks the given long array back into the single entries it holds. + * + * @param packed the packed representation to read + * @param entryCount the amount of entries the packed representation holds + * @param bitsPerEntry the amount of bits a single entry occupies + * @return the unpacked entries + * @throws IllegalArgumentException if the amount of bits per entry is not usable or if the + * packed array is too short for the requested entry count + */ + @Contract(pure = true) + public static int[] unpack(long[] packed, int entryCount, int bitsPerEntry) { + int required = expectedLongCount(entryCount, bitsPerEntry); + + if (packed.length < required) { + throw new IllegalArgumentException( + "The packed data holds " + packed.length + " longs but " + required + " are required" + ); + } + + int entriesPerLong = entriesPerLong(bitsPerEntry); + long mask = mask(bitsPerEntry); + int[] values = new int[entryCount]; + + for (int index = 0; index < entryCount; index++) { + int longIndex = index / entriesPerLong; + int bitOffset = (index % entriesPerLong) * bitsPerEntry; + values[index] = (int) ((packed[longIndex] >>> bitOffset) & mask); + } + return values; + } + + /** + * Derives the amount of bits per entry from the length of an already packed array. + * The format allows a writer to use more bits than the palette size requires, so the + * expected amount of bits is only a starting point and is verified against the length. + * + * @param longCount the amount of longs the packed representation holds + * @param entryCount the amount of entries the packed representation holds + * @param expectedBitsPerEntry the amount of bits which the palette size suggests + * @return the amount of bits per entry or zero if no amount matches the given length + */ + @Contract(pure = true) + public static int resolveBitsPerEntry(int longCount, int entryCount, int expectedBitsPerEntry) { + if (expectedBitsPerEntry > 0 && expectedLongCount(entryCount, expectedBitsPerEntry) == longCount) { + return expectedBitsPerEntry; + } + + for (int candidate = 1; candidate <= BITS_PER_LONG; candidate++) { + if (expectedLongCount(entryCount, candidate) == longCount) { + return candidate; + } + } + return 0; + } + + /** + * Calculates how many entries fit into a single long. + * + * @param bitsPerEntry the amount of bits a single entry occupies + * @return the amount of entries per long + * @throws IllegalArgumentException if the amount of bits per entry is not usable + */ + @Contract(pure = true) + private static int entriesPerLong(int bitsPerEntry) { + if (bitsPerEntry <= 0 || bitsPerEntry > BITS_PER_LONG) { + throw new IllegalArgumentException("The amount of bits per entry must be within [1, 64] but was " + bitsPerEntry); + } + return BITS_PER_LONG / bitsPerEntry; + } + + /** + * Creates the bit mask which isolates a single entry. + * + * @param bitsPerEntry the amount of bits a single entry occupies + * @return the mask for a single entry + */ + @Contract(pure = true) + private static long mask(int bitsPerEntry) { + return bitsPerEntry == BITS_PER_LONG ? -1L : (1L << bitsPerEntry) - 1L; + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/BlockPaletteResolver.java b/src/main/java/net/theevilreaper/aves/instance/anvil/BlockPaletteResolver.java new file mode 100644 index 00000000..6a965685 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/BlockPaletteResolver.java @@ -0,0 +1,120 @@ +package net.theevilreaper.aves.instance.anvil; + +import net.kyori.adventure.nbt.BinaryTag; +import net.kyori.adventure.nbt.CompoundBinaryTag; +import net.kyori.adventure.nbt.StringBinaryTag; +import net.minestom.server.instance.block.Block; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +/** + * The {@link BlockPaletteResolver} class translates between the block entries of the Anvil format + * and the block state ids of Minestom. + *

+ * An entry the server does not know is replaced with air instead of failing. A world can hold + * blocks of a mod or of a newer game version and rejecting the whole chunk over a single unknown + * block would lose far more data than it protects. Every replaced name is reported once through + * the diagnostics so the problem stays visible without flooding the log. + *

+ * + *

+ * This type is experimental. The Anvil loader is new and its API may still change while it is + * being validated against real worlds. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class BlockPaletteResolver implements PaletteEntryResolver { + + private static final Logger LOGGER = LoggerFactory.getLogger(BlockPaletteResolver.class); + + private static final String NAME_KEY = "Name"; + private static final String PROPERTIES_KEY = "Properties"; + + private final AnvilDiagnostics diagnostics; + + /** + * Creates a new resolver which reports unknown blocks to the given diagnostics. + * + * @param diagnostics the diagnostics which throttle the reports + */ + public BlockPaletteResolver(AnvilDiagnostics diagnostics) { + this.diagnostics = diagnostics; + } + + /** + * {@inheritDoc} + */ + @Override + public int toId(String name, @Nullable CompoundBinaryTag properties) { + Block block = Block.fromKey(name); + + if (block == null) { + if (this.diagnostics.reportUnknownBlock(name)) { + LOGGER.warn("The block '{}' is unknown and is replaced with air, further chunks with it are not reported", name); + } + return Block.AIR.stateId(); + } + if (properties == null || properties.size() == 0) { + return block.stateId(); + } + return block.withProperties(readProperties(name, properties)).stateId(); + } + + /** + * {@inheritDoc} + */ + @Override + public CompoundBinaryTag toEntry(int id) { + Block block = Block.fromStateId(id); + + if (block == null) { + return CompoundBinaryTag.builder().putString(NAME_KEY, Block.AIR.key().asString()).build(); + } + + CompoundBinaryTag.Builder entry = CompoundBinaryTag.builder().putString(NAME_KEY, block.key().asString()); + Map properties = block.properties(); + + if (!properties.isEmpty()) { + CompoundBinaryTag.Builder values = CompoundBinaryTag.builder(); + properties.forEach(values::putString); + entry.put(PROPERTIES_KEY, values.build()); + } + return entry.build(); + } + + /** + * Reads the properties of a palette entry. + * A property which does not hold a string is skipped because the format only defines string + * values for them. + * + * @param name the name of the block the properties belong to + * @param properties the properties of the palette entry + * @return the properties of the block + */ + private Map readProperties(String name, CompoundBinaryTag properties) { + Map values = HashMap.newHashMap(properties.size()); + + for (Map.Entry property : properties) { + if (property.getValue() instanceof StringBinaryTag value) { + values.put(property.getKey(), value.value()); + continue; + } + if (this.diagnostics.reportUnknownBlock(name + "#" + property.getKey())) { + LOGGER.warn( + "The property '{}' of the block '{}' is a {} instead of a string and is skipped", + property.getKey(), name, property.getValue().type() + ); + } + } + return values; + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/ChunkCompression.java b/src/main/java/net/theevilreaper/aves/instance/anvil/ChunkCompression.java new file mode 100644 index 00000000..17df0768 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/ChunkCompression.java @@ -0,0 +1,230 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import java.util.zip.InflaterInputStream; + +/** + * The {@link ChunkCompression} enum describes the compression schemes which a chunk payload + * inside a region file can use. The scheme is stored as a single byte in front of the payload. + *

+ * A scheme with the {@link #EXTERNAL_FLAG} set marks a chunk which does not live inside the + * region file itself but in a separate file next to it. The flag only describes the storage + * location, the remaining bits still name the compression of the payload. + *

+ * + *

+ * This type is experimental. The Anvil loader is new and its API may still change while it is + * being validated against real worlds. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public enum ChunkCompression { + + /** + * The payload is compressed with gzip. + */ + GZIP(1), + + /** + * The payload is compressed with zlib. This is the scheme which vanilla writes by default. + */ + ZLIB(2), + + /** + * The payload is stored without any compression. + */ + NONE(3); + + /** + * The bit which marks a chunk that is stored in a separate file next to the region file. + */ + public static final int EXTERNAL_FLAG = 0x80; + + /** + * The fastest compression level. It produces roughly a tenth more bytes than the default one. + */ + public static final int FASTEST_LEVEL = 1; + + /** + * The level this loader uses unless another one is chosen. + *

+ * It sits below the default of the platform on purpose. Compression is the largest single cost + * of saving a chunk, and on a serialised 24 section chunk the platform default spends about + * eighty percent longer to produce a result about three percent smaller. Levels between this + * one and the platform default are not worth choosing: they cost nearly as much as the default + * while saving almost nothing over it. + *

+ *

+ * A caller that stores a world once and reads it many times should pass a higher level + * explicitly. This default is chosen for the opposite case, where chunks are written repeatedly + * while a server runs. + *

+ */ + public static final int DEFAULT_LEVEL = 2; + + /** + * The level which produces the smallest result. It is far slower than every other level. + */ + public static final int SMALLEST_LEVEL = 9; + + private static final int BUFFER_SIZE = 8192; + + private final int id; + + /** + * Creates a new compression scheme with the identifier the format defines for it. + * + * @param id the identifier of the scheme inside a region file + */ + ChunkCompression(int id) { + this.id = id; + } + + /** + * Resolves the compression scheme which belongs to the given identifier. + * A set {@link #EXTERNAL_FLAG} is stripped before the lookup happens. + * + * @param id the identifier to resolve + * @return the matching compression scheme + * @throws IOException if no supported scheme uses the given identifier + */ + public static ChunkCompression fromId(int id) throws IOException { + return switch (id & ~EXTERNAL_FLAG) { + case 1 -> GZIP; + case 2 -> ZLIB; + case 3 -> NONE; + default -> throw new IOException( + "The compression scheme " + id + " is not supported. Only gzip (1), zlib (2) and none (3) can be read" + ); + }; + } + + /** + * Checks whether the given identifier marks a chunk which is stored outside of the region file. + * + * @param id the identifier to check + * @return true if the chunk is stored externally, otherwise false + */ + @Contract(pure = true) + public static boolean isExternal(int id) { + return (id & EXTERNAL_FLAG) != 0; + } + + /** + * Returns the identifier which the format uses for this scheme. + * + * @return the identifier of the scheme + */ + @Contract(pure = true) + public int id() { + return this.id; + } + + /** + * Compresses the given payload with this scheme. + * + * @param payload the uncompressed payload + * @return the compressed payload + * @throws IOException if the payload cannot be compressed + */ + public byte[] compress(byte[] payload) throws IOException { + return compress(payload, DEFAULT_LEVEL); + } + + /** + * Compresses the given payload with this scheme at the given level. + *

+ * A higher level spends more time to produce fewer bytes. The relation is far from linear: past + * the middle of the range the extra time grows steeply while the saved bytes do not, so the + * highest levels are rarely worth their cost for chunk data. + *

+ * + * @param payload the uncompressed payload + * @param level the compression level between {@link #FASTEST_LEVEL} and {@link #SMALLEST_LEVEL} + * @return the compressed payload + * @throws IOException if the payload cannot be compressed + * @throws IllegalArgumentException if the level is outside of the allowed range + */ + public byte[] compress(byte[] payload, int level) throws IOException { + if (level < FASTEST_LEVEL || level > SMALLEST_LEVEL) { + throw new IllegalArgumentException( + "The compression level must be within [" + FASTEST_LEVEL + ", " + SMALLEST_LEVEL + "] but was " + level + ); + } + if (this == NONE) { + return payload.clone(); + } + + ByteArrayOutputStream target = new ByteArrayOutputStream(Math.max(payload.length / 4, BUFFER_SIZE)); + + if (this == GZIP) { + // The gzip stream owns its deflater and ends it on close, so the level is set on that + // one rather than on a deflater of our own. + try (GZIPOutputStream stream = new GZIPOutputStream(target, BUFFER_SIZE) { + { + this.def.setLevel(level); + } + }) { + stream.write(payload); + } + return target.toByteArray(); + } + + Deflater deflater = new Deflater(level); + + try (OutputStream stream = new DeflaterOutputStream(target, deflater)) { + stream.write(payload); + } finally { + // A deflater holds native memory which the stream does not release on its own. + deflater.end(); + } + return target.toByteArray(); + } + + /** + * Decompresses the given payload with this scheme. + * + * @param payload the compressed payload + * @return the uncompressed payload + * @throws IOException if the payload cannot be decompressed + */ + public byte[] decompress(byte[] payload) throws IOException { + if (this == NONE) { + return payload.clone(); + } + + try (InputStream stream = wrapForDecompression(new ByteArrayInputStream(payload))) { + return stream.readAllBytes(); + } + } + + /** + * Wraps the given source stream into the decompressing stream of this scheme. + * + * @param source the stream which holds the compressed bytes + * @return the wrapped stream + * @throws IOException if the wrapping stream cannot be created + */ + private InputStream wrapForDecompression(InputStream source) throws IOException { + return switch (this) { + case GZIP -> new GZIPInputStream(source, BUFFER_SIZE); + case ZLIB -> new InflaterInputStream(source); + case NONE -> source; + }; + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/NbtReads.java b/src/main/java/net/theevilreaper/aves/instance/anvil/NbtReads.java new file mode 100644 index 00000000..f647b886 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/NbtReads.java @@ -0,0 +1,222 @@ +package net.theevilreaper.aves.instance.anvil; + +import net.kyori.adventure.nbt.BinaryTag; +import net.kyori.adventure.nbt.BinaryTagType; +import net.kyori.adventure.nbt.BinaryTagTypes; +import net.kyori.adventure.nbt.CompoundBinaryTag; +import net.kyori.adventure.nbt.IntArrayBinaryTag; +import net.kyori.adventure.nbt.ListBinaryTag; +import net.kyori.adventure.nbt.LongArrayBinaryTag; +import net.kyori.adventure.nbt.NumberBinaryTag; +import net.kyori.adventure.nbt.StringBinaryTag; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.Nullable; + +import java.io.IOException; + +/** + * The {@link NbtReads} class provides strict read access to Adventure NBT structures. + *

+ * The getters of {@link CompoundBinaryTag} return a default value when a key is missing or holds + * an unexpected type. For chunk data that behaviour is dangerous because a broken region file + * would silently turn into an empty chunk which overwrites the real data on the next save. + * Every method of this class therefore reports a missing or mistyped value as an error. + *

+ *

+ * The class also avoids the iterators of the array tags. In Adventure 5.1.1 those iterators stop + * one entry early which would drop the last entry of every packed block or biome array. + *

+ * + *

+ * This type is experimental. The Anvil loader is new and its API may still change while it is + * being validated against real worlds. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class NbtReads { + + private NbtReads() { + } + + /** + * Reads a long array and copies every entry of it. + * + * @param compound the compound which holds the array + * @param key the key of the array + * @return the entries of the array + * @throws IOException if the key is missing or does not hold a long array + */ + public static long[] longArray(CompoundBinaryTag compound, String key) throws IOException { + if (!(compound.get(key) instanceof LongArrayBinaryTag tag)) { + throw missing(compound, key, "a long array"); + } + + long[] values = new long[tag.size()]; + + for (int index = 0; index < values.length; index++) { + values[index] = tag.get(index); + } + return values; + } + + /** + * Reads an int array and copies every entry of it. + * + * @param compound the compound which holds the array + * @param key the key of the array + * @return the entries of the array + * @throws IOException if the key is missing or does not hold an int array + */ + public static int[] intArray(CompoundBinaryTag compound, String key) throws IOException { + if (!(compound.get(key) instanceof IntArrayBinaryTag tag)) { + throw missing(compound, key, "an int array"); + } + + int[] values = new int[tag.size()]; + + for (int index = 0; index < values.length; index++) { + values[index] = tag.get(index); + } + return values; + } + + /** + * Reads a nested compound. + * + * @param compound the compound which holds the nested compound + * @param key the key of the nested compound + * @return the nested compound + * @throws IOException if the key is missing or does not hold a compound + */ + public static CompoundBinaryTag compound(CompoundBinaryTag compound, String key) throws IOException { + if (!(compound.get(key) instanceof CompoundBinaryTag tag)) { + throw missing(compound, key, "a compound"); + } + return tag; + } + + /** + * Reads a nested compound which is allowed to be absent. + * + * @param compound the compound which holds the nested compound + * @param key the key of the nested compound + * @return the nested compound or null if the key is absent or holds another type + */ + @Contract(pure = true) + public static @Nullable CompoundBinaryTag optionalCompound(CompoundBinaryTag compound, String key) { + return compound.get(key) instanceof CompoundBinaryTag tag ? tag : null; + } + + /** + * Reads a list and verifies the type of its elements. + * An empty list always reports {@link BinaryTagTypes#END} as its element type, so it is + * accepted for every requested type. + * + * @param compound the compound which holds the list + * @param key the key of the list + * @param elementType the type every element of the list has to use + * @return the list + * @throws IOException if the key is missing, does not hold a list or holds other elements + */ + public static ListBinaryTag list(CompoundBinaryTag compound, String key, BinaryTagType elementType) throws IOException { + if (!(compound.get(key) instanceof ListBinaryTag tag)) { + throw missing(compound, key, "a list"); + } + if (tag.size() > 0 && tag.elementType() != elementType) { + throw new IOException("The key '" + key + "' holds a list of another element type than the expected one"); + } + return tag; + } + + /** + * Reads a list which is allowed to be absent. + * + * @param compound the compound which holds the list + * @param key the key of the list + * @param elementType the type every element of the list has to use + * @return the list or an empty list if the key is absent or holds another type + */ + @Contract(pure = true) + public static ListBinaryTag optionalList(CompoundBinaryTag compound, String key, BinaryTagType elementType) { + if (compound.get(key) instanceof ListBinaryTag tag && (tag.size() == 0 || tag.elementType() == elementType)) { + return tag; + } + return ListBinaryTag.empty(); + } + + /** + * Reads a string value. + * + * @param compound the compound which holds the value + * @param key the key of the value + * @return the value + * @throws IOException if the key is missing or does not hold a string + */ + public static String string(CompoundBinaryTag compound, String key) throws IOException { + if (!(compound.get(key) instanceof StringBinaryTag tag)) { + throw missing(compound, key, "a string"); + } + return tag.value(); + } + + /** + * Reads a string value which is allowed to be absent. + * + * @param compound the compound which holds the value + * @param key the key of the value + * @return the value or null if the key is absent or holds another type + */ + @Contract(pure = true) + public static @Nullable String optionalString(CompoundBinaryTag compound, String key) { + return compound.get(key) instanceof StringBinaryTag tag ? tag.value() : null; + } + + /** + * Reads a numeric value as an int. Every numeric tag is accepted because the format stores + * some values with a narrower type than an int. + * + * @param compound the compound which holds the value + * @param key the key of the value + * @return the value + * @throws IOException if the key is missing or does not hold a number + */ + public static int integer(CompoundBinaryTag compound, String key) throws IOException { + if (!(compound.get(key) instanceof NumberBinaryTag tag)) { + throw missing(compound, key, "a number"); + } + return tag.intValue(); + } + + /** + * Reads a numeric value as an int which is allowed to be absent. + * + * @param compound the compound which holds the value + * @param key the key of the value + * @param defaultValue the value to use if the key is absent + * @return the value or the given default value + */ + @Contract(pure = true) + public static int optionalInteger(CompoundBinaryTag compound, String key, int defaultValue) { + return compound.get(key) instanceof NumberBinaryTag tag ? tag.intValue() : defaultValue; + } + + /** + * Builds the error for a key which is missing or holds an unexpected type. + * + * @param compound the compound which was read + * @param key the key which was requested + * @param expected the description of the expected type + * @return the error to report + */ + @Contract(pure = true, value = "_, _, _ -> new") + private static IOException missing(CompoundBinaryTag compound, String key, String expected) { + BinaryTag actual = compound.get(key); + String description = actual == null ? "is absent" : "holds " + actual.type(); + return new IOException("The key '" + key + "' " + description + " but " + expected + " was expected"); + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/PaletteData.java b/src/main/java/net/theevilreaper/aves/instance/anvil/PaletteData.java new file mode 100644 index 00000000..c86c5016 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/PaletteData.java @@ -0,0 +1,209 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.Nullable; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * The {@link PaletteData} record holds the palette of a section together with the packed indices + * which reference it. It is the representation the codec works with between a region file and the + * palettes of Minestom. + *

+ * The record is immutable and thread confined by usage. A loader thread builds it without any lock + * and hands it over to a Minestom palette afterwards, which gives the data a safe publication + * through the final fields of the record. + *

+ *

+ * A section in which every entry holds the same value carries no packed data at all. The format + * stores such a section with a palette of a single entry and without a data array. + *

+ * + * @param palette the distinct values of the section in the order the format stores them + * @param packed the packed palette indices or null if the section holds a single value + * @param bitsPerEntry the amount of bits a single index occupies + * @param entryCount the amount of entries the section holds + *

+ * This type is experimental. The Anvil loader is new and its API may still change while it is + * being validated against real worlds. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public record PaletteData(int[] palette, long @Nullable [] packed, int bitsPerEntry, int entryCount) { + + /** + * The marker a uniformity scan reports when a section holds more than one value. + * It is outside the range of a block state id, which is never negative. + */ + private static final int NOT_UNIFORM = Integer.MIN_VALUE; + + /** + * Creates a representation for a section in which every entry holds the same value. + * + * @param value the value every entry of the section holds + * @param entryCount the amount of entries the section holds + * @return the created representation + */ + @Contract(pure = true, value = "_, _ -> new") + public static PaletteData single(int value, int entryCount) { + return new PaletteData(new int[]{value}, null, 0, entryCount); + } + + /** + * Reads a palette which was stored in a region file. + *

+ * The amount of bits per entry is verified against the length of the packed data instead of + * being derived from the palette size alone. A writer is allowed to use more bits than the + * palette requires and deriving the value from the palette size would decode such data + * incorrectly. + *

+ * + * @param palette the distinct values of the section + * @param packed the packed palette indices or null if the section holds a single value + * @param entryCount the amount of entries the section holds + * @param minBitsPerEntry the smallest amount of bits the palette type allows + * @return the created representation + * @throws IOException if the palette is empty or the packed data has an unusable length + */ + public static PaletteData read(int[] palette, long @Nullable [] packed, int entryCount, int minBitsPerEntry) throws IOException { + if (palette.length == 0) { + throw new IOException("The palette of a section must hold at least one entry"); + } + if (packed == null || packed.length == 0) { + return single(palette[0], entryCount); + } + + int expected = BitPacker.bitsPerEntry(palette.length, minBitsPerEntry); + int resolved = BitPacker.resolveBitsPerEntry(packed.length, entryCount, expected); + + if (resolved == 0) { + throw new IOException( + "The packed data of a section holds " + packed.length + " longs which does not match any bit count for " + + entryCount + " entries" + ); + } + return new PaletteData(palette, packed, resolved, entryCount); + } + + /** + * Builds the representation for the given values by collecting the distinct ones into a + * palette and packing the indices which reference them. + * + * @param values the value of every entry of the section + * @param minBitsPerEntry the smallest amount of bits the palette type allows + * @return the created representation + */ + public static PaletteData encode(int[] values, int minBitsPerEntry) { + // Whole sections of a world hold one repeated state: air above the terrain, stone below it, + // water in an ocean. Building a palette map over thousands of identical entries only to + // collapse it again afterwards is the common case, so it is recognised first. The scan + // stops at the first differing entry, which makes it free for every other section. + int uniform = uniformValueOf(values); + + if (uniform != NOT_UNIFORM) { + return single(uniform, values.length); + } + + Map indices = new HashMap<>(); + int[] mapped = new int[values.length]; + + for (int i = 0; i < values.length; i++) { + mapped[i] = indices.computeIfAbsent(values[i], ignored -> indices.size()); + } + + int[] palette = new int[indices.size()]; + + for (Map.Entry entry : indices.entrySet()) { + palette[entry.getValue()] = entry.getKey(); + } + + if (palette.length == 1) { + return single(palette[0], values.length); + } + + int bitsPerEntry = BitPacker.bitsPerEntry(palette.length, minBitsPerEntry); + return new PaletteData(palette, BitPacker.pack(mapped, bitsPerEntry), bitsPerEntry, values.length); + } + + /** + * Determines whether every entry of the given section holds the same value. + * + * @param values the value of every entry of the section + * @return the repeated value, or {@link #NOT_UNIFORM} if the section holds more than one + */ + @Contract(pure = true) + private static int uniformValueOf(int[] values) { + if (values.length == 0) { + return NOT_UNIFORM; + } + + int first = values[0]; + + for (int value : values) { + if (value != first) { + return NOT_UNIFORM; + } + } + return first; + } + + /** + * Checks whether every entry of the section holds the same value. + * + * @return true if the section holds a single value, otherwise false + */ + @Contract(pure = true) + public boolean isSingleValue() { + return this.packed == null; + } + + /** + * Returns the value every entry of the section holds. + * + * @return the value of every entry + * @throws IllegalStateException if the section does not hold a single value + */ + @Contract(pure = true) + public int singleValue() { + if (!isSingleValue()) { + throw new IllegalStateException("The section holds " + this.palette.length + " distinct values"); + } + return this.palette[0]; + } + + /** + * Resolves the value of every entry of the section through the palette. + * + * @return the value of every entry + * @throws IOException if a packed index does not address an entry of the palette + */ + public int[] unpack() throws IOException { + int[] values = new int[this.entryCount]; + + if (isSingleValue()) { + java.util.Arrays.fill(values, this.palette[0]); + return values; + } + + int[] indices = BitPacker.unpack(this.packed, this.entryCount, this.bitsPerEntry); + + for (int i = 0; i < values.length; i++) { + int index = indices[i]; + + if (index < 0 || index >= this.palette.length) { + throw new IOException( + "The packed index " + index + " does not address one of the " + this.palette.length + " palette entries" + ); + } + values[i] = this.palette[index]; + } + return values; + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/PaletteEntryResolver.java b/src/main/java/net/theevilreaper/aves/instance/anvil/PaletteEntryResolver.java new file mode 100644 index 00000000..e33d00d6 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/PaletteEntryResolver.java @@ -0,0 +1,51 @@ +package net.theevilreaper.aves.instance.anvil; + +import net.kyori.adventure.nbt.CompoundBinaryTag; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Nullable; + +/** + * The {@link PaletteEntryResolver} interface translates between the named palette entries of the + * Anvil format and the numeric ids a server works with. + *

+ * The format stores a block as a name with an optional set of properties and a biome as a plain + * name, while Minestom addresses both through an id. Keeping that translation behind an interface + * separates the file format from the registries of a running server, which lets the codec be + * verified without starting one. + *

+ * + *

+ * This type is experimental. The Anvil loader is new and its API may still change while it is + * being validated against real worlds. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public interface PaletteEntryResolver { + + /** + * Resolves the id which belongs to the given palette entry. + *

+ * An implementation must not fail for an unknown name. A world can hold entries of a mod or of + * a newer game version and losing a whole chunk over a single unknown entry would destroy more + * data than it protects. An implementation is expected to return a replacement id instead and + * to report the name to the caller. + *

+ * + * @param name the name of the palette entry + * @param properties the properties of the palette entry or null if it carries none + * @return the id which belongs to the entry + */ + int toId(String name, @Nullable CompoundBinaryTag properties); + + /** + * Builds the palette entry which belongs to the given id. + * + * @param id the id to describe + * @return the palette entry of the id + */ + CompoundBinaryTag toEntry(int id); +} diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/RegionConstants.java b/src/main/java/net/theevilreaper/aves/instance/anvil/RegionConstants.java new file mode 100644 index 00000000..35e01fa4 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/RegionConstants.java @@ -0,0 +1,127 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; + +/** + * The {@link RegionConstants} class holds all layout constants of the Anvil region file format. + * A region file starts with a two sector header. The first sector contains the location table + * and the second one the timestamp table. Both tables have an entry for each of the + * {@code 32 x 32} chunks a region can hold. + *

+ * The class is not meant to be instantiated. It only provides constants and pure helper methods. + *

+ * + *

+ * This type is experimental. The Anvil loader is new and its API may still change while it is + * being validated against real worlds. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class RegionConstants { + + /** + * The size of a single sector in bytes. + */ + public static final int SECTOR_SIZE = 4096; + + /** + * The amount of chunks a region file can address in a single axis. + */ + public static final int REGION_SIZE = 32; + + /** + * The amount of chunk entries a region file can address in total. + */ + public static final int ENTRY_COUNT = REGION_SIZE * REGION_SIZE; + + /** + * The amount of sectors which are reserved for the location and timestamp table. + */ + public static final int HEADER_SECTORS = 2; + + /** + * The size of the complete region file header in bytes. + */ + public static final int HEADER_SIZE = HEADER_SECTORS * SECTOR_SIZE; + + /** + * The highest amount of sectors a single chunk entry can address. + * The sector count is stored in a single byte which limits a chunk to roughly one mebibyte. + */ + public static final int MAX_SECTORS_PER_CHUNK = 255; + + /** + * The amount of bytes which are used to store the length of a chunk payload. + */ + public static final int LENGTH_FIELD_SIZE = Integer.BYTES; + + /** + * The amount of bytes which are used to store the compression scheme of a chunk payload. + */ + public static final int COMPRESSION_FIELD_SIZE = Byte.BYTES; + + private RegionConstants() { + } + + /** + * Calculates the index a chunk occupies inside the location and timestamp table. + * The given coordinates are absolute chunk coordinates and are wrapped into the region. + * + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @return the index of the chunk inside a region table + */ + @Contract(pure = true) + public static int index(int chunkX, int chunkZ) { + return ((chunkZ & (REGION_SIZE - 1)) << 5) | (chunkX & (REGION_SIZE - 1)); + } + + /** + * Calculates the byte offset of a chunk entry inside the location table. + * + * @param index the index of the chunk inside a region table + * @return the byte offset of the location entry + */ + @Contract(pure = true) + public static int locationOffset(int index) { + return index * Integer.BYTES; + } + + /** + * Calculates the byte offset of a chunk entry inside the timestamp table. + * + * @param index the index of the chunk inside a region table + * @return the byte offset of the timestamp entry + */ + @Contract(pure = true) + public static int timestampOffset(int index) { + return SECTOR_SIZE + index * Integer.BYTES; + } + + /** + * Converts an absolute chunk coordinate into the coordinate of the region which contains it. + * + * @param chunkCoordinate the absolute chunk coordinate + * @return the region coordinate + */ + @Contract(pure = true) + public static int chunkToRegion(int chunkCoordinate) { + return chunkCoordinate >> 5; + } + + /** + * Calculates the amount of sectors which are required to store the given amount of bytes. + * + * @param byteLength the amount of bytes to store + * @return the amount of sectors which are required + */ + @Contract(pure = true) + public static int sectorsFor(int byteLength) { + return (byteLength + SECTOR_SIZE - 1) / SECTOR_SIZE; + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java b/src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java new file mode 100644 index 00000000..1fdc364c --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/RegionFile.java @@ -0,0 +1,694 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.Nullable; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.AccessDeniedException; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.concurrent.locks.ReentrantLock; + +/** + * The {@link RegionFile} class represents a single Anvil region file which stores up to + * {@code 32 x 32} chunks. The class is a pure byte container. It neither knows the NBT structure + * of a chunk nor the Minestom chunk model which keeps the file format concern isolated. + *

+ * Reading uses positional channel operations which do not touch the channel position and are + * therefore safe to run from multiple threads at the same time. Only the sector allocation, the + * header update and the switch between the two storage locations of a chunk need the internal lock, + * so the expensive work of a caller stays outside of any critical section. + *

+ *

+ * A reader still has to notice when the bytes below it changed while it read them. A chunk which is + * rewritten moves to a different sector range and releases the one it occupied, and the allocator + * may hand that range to the very next write of any chunk. A reader which took no lock can therefore + * be somewhere inside a range which now belongs to a different chunk. Every chunk entry carries a + * version counter for that reason. The counter is raised once when a writer enters its critical + * section and once when it leaves it, so an odd value marks an entry which is currently being + * changed. A reader takes the counter, rejects an odd one, reads the bytes and takes the counter + * again: an unchanged even counter proves that no writer touched this chunk in between, and a range + * can only be recycled after the chunk which owned it was rewritten. Everything else makes the + * reader start over. Readers therefore never block each other and never block a writer, which is the + * property the whole design rests on. + *

+ *

+ * A chunk which does not fit into the {@link RegionConstants#MAX_SECTORS_PER_CHUNK} sectors a + * location entry can address is moved into a separate file next to the region file. The header entry + * decides which of the two locations a reader has to follow, so the file and the entry are switched + * inside the same critical section. Only the payload bytes of such a chunk are written outside of + * it, into a staging file which is moved into place while the lock is held. + *

+ *

+ * The external file is the one place where the lock free reads meet a name in the file system + * instead of a range inside the region file, and a name is not a POSIX concept. A reader opens the + * external file while a writer may replace or remove it, which POSIX allows without any further + * thought but Windows does not. Every operation on such a file therefore goes through + * {@link #placeExternal(Path, Path)} and {@link #removeExternal(Path)} which keep the name usable + * for a writer while a reader still holds a handle on the file behind it. + *

+ * + *

+ * This type is experimental. The Anvil loader is new and its API may still change while it is + * being validated against real worlds. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class RegionFile implements AutoCloseable { + + /** + * The amount of times a read is repeated before it falls back to the lock. + *

+ * A repetition only happens when a writer touched the very chunk which is being read, which is + * rare enough that a handful of attempts practically always succeed. The fallback exists so a + * chunk which is rewritten in a tight loop cannot starve a reader forever. It is the only case + * in which a reader waits for a writer at all. + *

+ */ + private static final int OPTIMISTIC_ATTEMPTS = 4; + + /** + * The suffix of the file which holds the payload of an oversized chunk until it is moved into + * place. The suffix differs from the one of a finished external file so a reader can never pick + * up a staging file by name. + */ + private static final String STAGING_SUFFIX = ".mcc.tmp"; + + /** + * The amount of times an operation on an external chunk file is repeated before it gives up. + *

+ * A repetition only happens on a file system which refuses to touch a name while another thread + * has the file behind it open. The refusal lasts exactly as long as that handle, and a handle on + * an external file only exists for the duration of a single {@link Files#readAllBytes(Path)} in + * {@link #readEntry(int, int, int)}. No reader can open a new one while the writer holds the + * lock, because the version counter of the entry is odd for that whole time and makes every + * reader either spin or wait for the lock. The outstanding handles therefore drain within one + * read, and the limit only exists so a file which is held open by something outside of this + * process reports a failure instead of blocking a writer forever. + *

+ */ + private static final int EXTERNAL_ATTEMPTS = 100; + + /** + * The time in milliseconds a thread waits before it repeats an operation on an external chunk + * file. + */ + private static final long EXTERNAL_RETRY_DELAY = 1L; + + private final Path path; + private final Path directory; + private final FileChannel channel; + private final ReentrantLock lock; + private final AtomicIntegerArray locations; + private final AtomicIntegerArray timestamps; + private final AtomicIntegerArray versions; + private final SectorAllocator allocator; + + private volatile boolean closed; + + /** + * Creates a new region file around the given channel and header state. + * + * @param path the path of the region file + * @param channel the channel which is used for all read and write operations + * @param locations the location table of the region file + * @param timestamps the timestamp table of the region file + * @param allocator the allocator which tracks the sector usage + */ + private RegionFile(Path path, FileChannel channel, int[] locations, int[] timestamps, SectorAllocator allocator) { + this.path = path; + this.directory = path.getParent() == null ? Path.of(".") : path.getParent(); + this.channel = channel; + this.lock = new ReentrantLock(); + this.locations = new AtomicIntegerArray(locations); + this.timestamps = new AtomicIntegerArray(timestamps); + this.versions = new AtomicIntegerArray(RegionConstants.ENTRY_COUNT); + this.allocator = allocator; + } + + /** + * Opens the region file under the given path and reads its header. + * A file which does not exist yet is created with an empty header. + * + * @param path the path of the region file + * @return the opened region file + * @throws IOException if the file cannot be opened or holds a broken header + */ + public static RegionFile open(Path path) throws IOException { + Path parent = path.getParent(); + + if (parent != null) { + Files.createDirectories(parent); + } + + FileChannel channel = FileChannel.open( + path, StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE + ); + + try { + return readHeader(path, channel); + } catch (IOException | RuntimeException exception) { + channel.close(); + throw exception; + } + } + + /** + * Reads the header of an already opened region file and rebuilds the sector usage from it. + * + * @param path the path of the region file + * @param channel the channel of the region file + * @return the region file which is described by the header + * @throws IOException if the header is incomplete or describes an invalid layout + */ + private static RegionFile readHeader(Path path, FileChannel channel) throws IOException { + long size = channel.size(); + int[] locations = new int[RegionConstants.ENTRY_COUNT]; + int[] timestamps = new int[RegionConstants.ENTRY_COUNT]; + + if (size == 0) { + channel.write(ByteBuffer.allocate(RegionConstants.HEADER_SIZE), 0); + return new RegionFile(path, channel, locations, timestamps, new SectorAllocator(RegionConstants.HEADER_SECTORS)); + } + + if (size < RegionConstants.HEADER_SIZE) { + throw new IOException( + "The region file " + path + " holds " + size + " bytes which is less than the header size of " + + RegionConstants.HEADER_SIZE + " bytes" + ); + } + + ByteBuffer header = readFully(channel, 0, RegionConstants.HEADER_SIZE, path); + + for (int index = 0; index < RegionConstants.ENTRY_COUNT; index++) { + locations[index] = header.getInt(RegionConstants.locationOffset(index)); + timestamps[index] = header.getInt(RegionConstants.timestampOffset(index)); + } + + int totalSectors = (int) Math.max(size / RegionConstants.SECTOR_SIZE, RegionConstants.HEADER_SECTORS); + SectorAllocator allocator = new SectorAllocator(totalSectors); + + for (int index = 0; index < RegionConstants.ENTRY_COUNT; index++) { + int location = locations[index]; + + if (location == 0) { + continue; + } + + int offset = location >>> 8; + int count = location & 0xFF; + + if (offset < RegionConstants.HEADER_SECTORS || count <= 0) { + locations[index] = 0; + continue; + } + allocator.reserve(offset, count); + } + return new RegionFile(path, channel, locations, timestamps, allocator); + } + + /** + * Reads the raw payload of the given chunk without decompressing it. + * The caller is expected to decompress the payload outside of any lock this class holds. + *

+ * The read takes no lock and is validated against the version counter of the chunk afterwards. + * A read which raced a writer is repeated, and a read which keeps racing falls back to the lock + * which the writers use, so it cannot be starved. A failure of the read itself is only reported + * when the version counter proves that no writer was involved, because the bytes of a recycled + * sector range can describe any length and any compression scheme. + *

+ * + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @return the raw chunk or null if the region file does not hold the chunk + * @throws IOException if the chunk cannot be read or holds an invalid header + */ + public @Nullable RawChunk readRaw(int chunkX, int chunkZ) throws IOException { + ensureOpen(); + int index = RegionConstants.index(chunkX, chunkZ); + + for (int attempt = 0; attempt < OPTIMISTIC_ATTEMPTS; attempt++) { + int version = this.versions.get(index); + + // An odd counter marks an entry which is being changed right now. The change is not + // limited to the tables: the external file of the chunk may already be gone while the + // entry still points at it, so such a read cannot be trusted at all. + if ((version & 1) != 0) { + continue; + } + + try { + RawChunk chunk = readEntry(index, chunkX, chunkZ); + + if (this.versions.get(index) == version) { + return chunk; + } + } catch (IOException exception) { + if (this.versions.get(index) == version) { + throw exception; + } + } + } + + this.lock.lock(); + try { + ensureOpen(); + return readEntry(index, chunkX, chunkZ); + } finally { + this.lock.unlock(); + } + } + + /** + * Reads the payload the location entry of the given index currently points at. + *

+ * The method performs no validation of its own result. A caller which did not take the lock has + * to confirm through the version counter of the chunk that the entry did not change while the + * bytes were read. + *

+ * + * @param index the index of the chunk inside the region tables + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @return the raw chunk or null if the region file does not hold the chunk + * @throws IOException if the chunk cannot be read or holds an invalid header + */ + private @Nullable RawChunk readEntry(int index, int chunkX, int chunkZ) throws IOException { + int location = this.locations.get(index); + + if (location == 0) { + return null; + } + + int sectorOffset = location >>> 8; + int sectorCount = location & 0xFF; + long position = (long) sectorOffset * RegionConstants.SECTOR_SIZE; + int available = sectorCount * RegionConstants.SECTOR_SIZE; + + ByteBuffer head = readFully(this.channel, position, RegionConstants.LENGTH_FIELD_SIZE + RegionConstants.COMPRESSION_FIELD_SIZE, this.path); + int length = head.getInt(); + int scheme = head.get() & 0xFF; + + if (length <= 0 || length > available) { + throw new IOException( + "The chunk " + chunkX + "/" + chunkZ + " in " + this.path + " declares a length of " + length + + " bytes which does not fit into its " + sectorCount + " sectors" + ); + } + + ChunkCompression compression = ChunkCompression.fromId(scheme); + + if (ChunkCompression.isExternal(scheme)) { + return new RawChunk(compression, Files.readAllBytes(externalPath(chunkX, chunkZ))); + } + + int payloadLength = length - RegionConstants.COMPRESSION_FIELD_SIZE; + ByteBuffer payload = readFully( + this.channel, position + RegionConstants.LENGTH_FIELD_SIZE + RegionConstants.COMPRESSION_FIELD_SIZE, + payloadLength, this.path + ); + byte[] bytes = new byte[payloadLength]; + payload.get(bytes); + return new RawChunk(compression, bytes); + } + + /** + * Writes the raw payload of the given chunk into the region file. + * The payload is expected to be compressed already so the compression can happen outside of + * the lock this method acquires. + *

+ * An oversized payload is written into a staging file before the lock is taken and only moved + * into its final place while the lock is held. The header entry and the external file therefore + * always describe the same storage location, no matter how two writers of the same chunk + * interleave, while the bytes still leave the process outside of the critical section. + *

+ * + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @param compression the compression scheme of the payload + * @param payload the compressed payload of the chunk + * @throws IOException if the chunk cannot be written + */ + public void writeRaw(int chunkX, int chunkZ, ChunkCompression compression, byte[] payload) throws IOException { + ensureOpen(); + int index = RegionConstants.index(chunkX, chunkZ); + int totalLength = RegionConstants.LENGTH_FIELD_SIZE + RegionConstants.COMPRESSION_FIELD_SIZE + payload.length; + boolean external = RegionConstants.sectorsFor(totalLength) > RegionConstants.MAX_SECTORS_PER_CHUNK; + Path staged = external ? Files.createTempFile(this.directory, "c.", STAGING_SUFFIX) : null; + + try { + if (staged != null) { + Files.write(staged, payload); + } + + byte[] stored = external ? new byte[0] : payload; + int scheme = external ? compression.id() | ChunkCompression.EXTERNAL_FLAG : compression.id(); + // The specification defines the length field as the compression byte plus the payload. + int length = RegionConstants.COMPRESSION_FIELD_SIZE + stored.length; + int sectorCount = RegionConstants.sectorsFor(RegionConstants.LENGTH_FIELD_SIZE + length); + + ByteBuffer buffer = ByteBuffer.allocate(sectorCount * RegionConstants.SECTOR_SIZE); + buffer.putInt(length).put((byte) scheme).put(stored); + buffer.rewind(); + + Path externalPath = externalPath(chunkX, chunkZ); + + this.lock.lock(); + try { + // The counter turns odd before the first change becomes visible and even again once + // every change is done, so a reader can tell a finished state from a state which is + // still being assembled. + this.versions.incrementAndGet(index); + + int previous = this.locations.get(index); + int sectorOffset = this.allocator.allocate(sectorCount); + + writeFully(this.channel, buffer, (long) sectorOffset * RegionConstants.SECTOR_SIZE); + + // The external file has to exist before the entry points at it and may only be + // removed after the entry stopped pointing at it, so a crash between the two steps + // can leave an unused file but never a missing one. + if (staged != null) { + placeExternal(staged, externalPath); + } + + this.locations.set(index, (sectorOffset << 8) | sectorCount); + this.timestamps.set(index, (int) (System.currentTimeMillis() / 1000L)); + writeEntry(index); + + if (staged == null) { + removeExternal(externalPath); + } + if (previous != 0) { + this.allocator.free(previous >>> 8, previous & 0xFF); + } + } finally { + // The counter is raised while the lock is still held, so a range which was freed + // above cannot be handed to another writer before every reader can see the change. + this.versions.incrementAndGet(index); + this.lock.unlock(); + } + } finally { + if (staged != null) { + retryWhileDenied(() -> Files.deleteIfExists(staged)); + } + } + } + + /** + * Removes the given chunk from the region file. + * The sectors the chunk occupied become available for a later write. + * + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @throws IOException if the header cannot be updated + */ + public void delete(int chunkX, int chunkZ) throws IOException { + ensureOpen(); + int index = RegionConstants.index(chunkX, chunkZ); + + this.lock.lock(); + try { + this.versions.incrementAndGet(index); + int previous = this.locations.get(index); + + if (previous == 0) { + return; + } + + this.locations.set(index, 0); + this.timestamps.set(index, 0); + writeEntry(index); + removeExternal(externalPath(chunkX, chunkZ)); + this.allocator.free(previous >>> 8, previous & 0xFF); + } finally { + this.versions.incrementAndGet(index); + this.lock.unlock(); + } + } + + /** + * Checks whether the region file holds the given chunk. + * + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @return true if the chunk is present, otherwise false + */ + @Contract(pure = true) + public boolean hasChunk(int chunkX, int chunkZ) { + return this.locations.get(RegionConstants.index(chunkX, chunkZ)) != 0; + } + + /** + * Returns the path of the region file. + * + * @return the path of the region file + */ + @Contract(pure = true) + public Path path() { + return this.path; + } + + /** + * Forces all pending changes of the region file to the underlying storage. + * + * @throws IOException if the changes cannot be written + */ + public void flush() throws IOException { + ensureOpen(); + this.channel.force(false); + } + + /** + * {@inheritDoc} + */ + @Override + public void close() throws IOException { + this.lock.lock(); + try { + if (this.closed) { + return; + } + this.closed = true; + this.channel.close(); + } finally { + this.lock.unlock(); + } + } + + /** + * Writes the location and the timestamp entry of the given index into the header. + * Only the eight affected bytes are touched so a crash cannot destroy the whole header. + * + * @param index the index of the chunk inside the region tables + * @throws IOException if the entry cannot be written + */ + private void writeEntry(int index) throws IOException { + ByteBuffer location = ByteBuffer.allocate(Integer.BYTES).putInt(this.locations.get(index)).rewind(); + writeFully(this.channel, location, RegionConstants.locationOffset(index)); + + ByteBuffer timestamp = ByteBuffer.allocate(Integer.BYTES).putInt(this.timestamps.get(index)).rewind(); + writeFully(this.channel, timestamp, RegionConstants.timestampOffset(index)); + } + + /** + * Builds the path of the file which holds an oversized chunk. + * + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @return the path of the external chunk file + */ + @Contract(pure = true) + private Path externalPath(int chunkX, int chunkZ) { + return this.directory.resolve("c." + chunkX + "." + chunkZ + ".mcc"); + } + + /** + * Moves the staging file of an oversized chunk onto the external file of that chunk. + *

+ * The move replaces a file which a reader may have open at this very moment. POSIX lets a name + * be re-pointed at any time and keeps every open handle valid, so the move always succeeds + * there. Windows only agrees as long as the readers opened the file in a way which shares the + * deletion right, which the NIO file system provider does, and as long as the name itself is not + * poisoned. That is why {@link #removeExternal(Path)} exists, and the repetition here covers + * what is left: a handle which is still being torn down or a virus scanner which opened the file + * behind the back of this process both deny the move for a moment and let it through afterwards. + *

+ * + * @param staged the staging file which holds the payload of the chunk + * @param target the external file of the chunk + * @throws IOException if the staging file cannot be moved into place + */ + private void placeExternal(Path staged, Path target) throws IOException { + retryWhileDenied(() -> Files.move(staged, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE)); + } + + /** + * Removes the external file of a chunk which no longer needs one. + *

+ * The file is renamed onto a private name before it is deleted instead of being deleted where it + * lies. A plain deletion looks equivalent and is equivalent under POSIX, where the name is + * detached immediately and only the unnamed file lives on until the last reader closed it. + * Windows instead keeps the name in the directory and marks the file for deletion, and for as + * long as a reader holds it open every attempt to open that name or to move another file onto it + * fails with an {@link AccessDeniedException}. A writer which switches the same chunk back to an + * external payload right after would therefore be denied its move for as long as any reader is + * still busy with the old file, which is precisely the window this class is built to keep open. + * Renaming the file away detaches the name at once on both systems, so only the private name is + * left in that state and nobody ever asks for it again. + *

+ * + * @param target the external file of the chunk + * @throws IOException if the external file cannot be removed + */ + private void removeExternal(Path target) throws IOException { + if (!Files.exists(target)) { + return; + } + Path discarded = Files.createTempFile(this.directory, "c.", STAGING_SUFFIX); + + try { + retryWhileDenied(() -> Files.move(target, discarded, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE)); + } catch (NoSuchFileException _) { + // Another writer removed the file between the check above and the move, which is the + // outcome this method wants anyway. + } + retryWhileDenied(() -> Files.deleteIfExists(discarded)); + } + + /** + * Runs the given action and repeats it while the file system denies the access to a name. + *

+ * The action is repeated at most {@link #EXTERNAL_ATTEMPTS} times with a pause of + * {@link #EXTERNAL_RETRY_DELAY} milliseconds in between. A denial which outlives every attempt + * is reported to the caller, and a thread which is interrupted while it waits stops immediately + * and reports the denial which made it wait. + *

+ * + * @param action the action to run + * @throws IOException if the action keeps failing or fails for another reason + */ + private static void retryWhileDenied(FileAction action) throws IOException { + for (int attempt = 1; ; attempt++) { + try { + action.run(); + return; + } catch (AccessDeniedException exception) { + if (attempt >= EXTERNAL_ATTEMPTS) { + throw exception; + } + + try { + Thread.sleep(EXTERNAL_RETRY_DELAY); + } catch (InterruptedException interruption) { + Thread.currentThread().interrupt(); + throw exception; + } + } + } + } + + /** + * An operation on a file which may fail with an {@link IOException}. + */ + @FunctionalInterface + private interface FileAction { + + /** + * Runs the operation. + * + * @throws IOException if the operation fails + */ + void run() throws IOException; + } + + /** + * Verifies that the region file is still usable. + * + * @throws IOException if the region file is already closed + */ + private void ensureOpen() throws IOException { + if (this.closed) { + throw new IOException("The region file " + this.path + " is already closed"); + } + } + + /** + * Reads the requested amount of bytes from the given position. + * A channel is allowed to return fewer bytes than requested, so the read is repeated until + * the buffer is filled or the file ends. + * + * @param channel the channel to read from + * @param position the position to start reading at + * @param length the amount of bytes to read + * @param path the path which is used for the error message + * @return a buffer which holds the requested bytes and is ready to be read + * @throws IOException if the file ends before the requested amount of bytes was read + */ + private static ByteBuffer readFully(FileChannel channel, long position, int length, Path path) throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(length); + long offset = position; + + while (buffer.hasRemaining()) { + int read = channel.read(buffer, offset); + + if (read < 0) { + throw new IOException( + "The file " + path + " ended after " + buffer.position() + " of " + length + " expected bytes" + ); + } + offset += read; + } + return buffer.rewind(); + } + + /** + * Writes the complete buffer to the given position. + * + * @param channel the channel to write to + * @param buffer the buffer which holds the bytes to write + * @param position the position to start writing at + * @throws IOException if the bytes cannot be written + */ + private static void writeFully(FileChannel channel, ByteBuffer buffer, long position) throws IOException { + long offset = position; + + while (buffer.hasRemaining()) { + offset += channel.write(buffer, offset); + } + } + + /** + * The {@link RawChunk} record holds the untouched payload of a chunk together with the + * compression scheme which is required to decode it. + * + * @param compression the compression scheme of the payload + * @param payload the payload as it is stored on disk + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ + public record RawChunk(ChunkCompression compression, byte[] payload) { + + /** + * Decompresses the payload of the chunk. + * + * @return the decompressed payload + * @throws IOException if the payload cannot be decompressed + */ + public byte[] decompress() throws IOException { + return this.compression.decompress(this.payload); + } + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/SectionCodec.java b/src/main/java/net/theevilreaper/aves/instance/anvil/SectionCodec.java new file mode 100644 index 00000000..5230d179 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/SectionCodec.java @@ -0,0 +1,169 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.jetbrains.annotations.ApiStatus; + +import net.kyori.adventure.nbt.BinaryTag; +import net.kyori.adventure.nbt.BinaryTagTypes; +import net.kyori.adventure.nbt.CompoundBinaryTag; +import net.kyori.adventure.nbt.ListBinaryTag; +import net.kyori.adventure.nbt.LongArrayBinaryTag; +import net.kyori.adventure.nbt.StringBinaryTag; + +import java.io.IOException; + +/** + * The {@link SectionCodec} class converts between a palette container of the Anvil format and the + * {@link PaletteData} representation the loader works with. + *

+ * A palette container is the shape the format uses for the blocks and the biomes of a section. It + * holds a list of named entries and an optional array of packed indices which reference them. A + * container without the array describes a section in which every entry holds the same value. + *

+ *

+ * The class is stateless so it can be used from every thread that loads or saves a chunk. + *

+ * + *

+ * This type is experimental. The Anvil loader is new and its API may still change while it is + * being validated against real worlds. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class SectionCodec { + + private static final String PALETTE_KEY = "palette"; + private static final String DATA_KEY = "data"; + private static final String NAME_KEY = "Name"; + private static final String PROPERTIES_KEY = "Properties"; + + private SectionCodec() { + } + + /** + * Reads a palette container and resolves every entry of it. + * + * @param container the palette container to read + * @param resolver the resolver which translates the names into ids + * @param entryCount the amount of entries the section holds + * @param minBitsPerEntry the smallest amount of bits the palette type allows + * @return the palette representation of the container + * @throws IOException if the container is malformed + */ + public static PaletteData decode(CompoundBinaryTag container, PaletteEntryResolver resolver, int entryCount, int minBitsPerEntry) throws IOException { + ListBinaryTag entries = NbtReads.list(container, PALETTE_KEY, BinaryTagTypes.COMPOUND); + + if (entries.size() == 0) { + throw new IOException("The palette container holds an empty palette"); + } + + int[] palette = new int[entries.size()]; + + for (int index = 0; index < palette.length; index++) { + CompoundBinaryTag entry = entries.getCompound(index); + palette[index] = resolver.toId( + NbtReads.string(entry, NAME_KEY), + NbtReads.optionalCompound(entry, PROPERTIES_KEY) + ); + } + + BinaryTag data = container.get(DATA_KEY); + + if (data == null) { + return PaletteData.read(palette, null, entryCount, minBitsPerEntry); + } + if (!(data instanceof LongArrayBinaryTag)) { + throw new IOException("The palette container holds a data entry which is not a long array"); + } + return PaletteData.read(palette, NbtReads.longArray(container, DATA_KEY), entryCount, minBitsPerEntry); + } + + /** + * Reads a biome palette container and resolves every entry of it. + *

+ * The format stores the biome palette as a list of plain names while the block palette holds a + * compound with a name and optional properties, so both shapes need their own conversion. + *

+ * + * @param container the palette container to read + * @param resolver the resolver which translates the names into ids + * @param entryCount the amount of entries the section holds + * @param minBitsPerEntry the smallest amount of bits the palette type allows + * @return the palette representation of the container + * @throws IOException if the container is malformed + */ + public static PaletteData decodeBiomes(CompoundBinaryTag container, PaletteEntryResolver resolver, int entryCount, int minBitsPerEntry) throws IOException { + ListBinaryTag entries = NbtReads.list(container, PALETTE_KEY, BinaryTagTypes.STRING); + + if (entries.size() == 0) { + throw new IOException("The biome palette container holds an empty palette"); + } + + int[] palette = new int[entries.size()]; + + for (int index = 0; index < palette.length; index++) { + palette[index] = resolver.toId(entries.getString(index), null); + } + + BinaryTag data = container.get(DATA_KEY); + + if (data == null) { + return PaletteData.read(palette, null, entryCount, minBitsPerEntry); + } + if (!(data instanceof LongArrayBinaryTag)) { + throw new IOException("The biome palette container holds a data entry which is not a long array"); + } + return PaletteData.read(palette, NbtReads.longArray(container, DATA_KEY), entryCount, minBitsPerEntry); + } + + /** + * Writes the given palette representation into a biome palette container. + * + * @param data the palette representation to write + * @param resolver the resolver which describes the ids + * @return the created palette container + */ + public static CompoundBinaryTag encodeBiomes(PaletteData data, PaletteEntryResolver resolver) { + ListBinaryTag.Builder entries = ListBinaryTag.builder(BinaryTagTypes.STRING); + + for (int id : data.palette()) { + entries.add(StringBinaryTag.stringBinaryTag(NbtReads.optionalString(resolver.toEntry(id), NAME_KEY))); + } + + CompoundBinaryTag.Builder container = CompoundBinaryTag.builder().put(PALETTE_KEY, entries.build()); + long[] packed = data.packed(); + + if (packed != null) { + container.put(DATA_KEY, LongArrayBinaryTag.longArrayBinaryTag(packed)); + } + return container.build(); + } + + /** + * Writes the given palette representation into a palette container. + * A representation which holds a single value is written without a data array, which is the + * shape the format uses for a uniform section. + * + * @param data the palette representation to write + * @param resolver the resolver which describes the ids + * @return the created palette container + */ + public static CompoundBinaryTag encode(PaletteData data, PaletteEntryResolver resolver) { + ListBinaryTag.Builder entries = ListBinaryTag.builder(BinaryTagTypes.COMPOUND); + + for (int id : data.palette()) { + entries.add(resolver.toEntry(id)); + } + + CompoundBinaryTag.Builder container = CompoundBinaryTag.builder().put(PALETTE_KEY, entries.build()); + long[] packed = data.packed(); + + if (packed != null) { + container.put(DATA_KEY, LongArrayBinaryTag.longArrayBinaryTag(packed)); + } + return container.build(); + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/SectorAllocator.java b/src/main/java/net/theevilreaper/aves/instance/anvil/SectorAllocator.java new file mode 100644 index 00000000..1ebe9bd7 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/SectorAllocator.java @@ -0,0 +1,130 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.jetbrains.annotations.Contract; + +import java.util.BitSet; + +/** + * The {@link SectorAllocator} tracks which sectors of a region file are currently in use. + * It hands out sector ranges with a first fit strategy and reuses ranges which were freed + * before. When no gap is large enough the allocator grows the region file virtually by + * returning a range behind the current end. + *

+ * The class is deliberately free of any file access so the allocation logic can be tested + * on its own. Instances are not thread safe and must be guarded by the owning region file. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +final class SectorAllocator { + + private final BitSet usedSectors; + private int totalSectors; + + /** + * Creates a new allocator which considers the first sectors as permanently used. + * The header of a region file always occupies {@link RegionConstants#HEADER_SECTORS} sectors. + * + * @param totalSectors the amount of sectors the region file currently spans + */ + SectorAllocator(int totalSectors) { + int sectors = Math.max(totalSectors, RegionConstants.HEADER_SECTORS); + this.usedSectors = new BitSet(sectors); + this.usedSectors.set(0, RegionConstants.HEADER_SECTORS); + this.totalSectors = sectors; + } + + /** + * Allocates a consecutive range of sectors and marks it as used. + * The allocator reuses a freed gap when one is large enough, otherwise the returned range + * starts behind the current end of the region file. + * + * @param count the amount of sectors to allocate + * @return the index of the first sector of the allocated range + * @throws IllegalArgumentException if the given count is not positive + */ + int allocate(int count) { + if (count <= 0) { + throw new IllegalArgumentException("The sector count must be positive but was " + count); + } + + int candidate = this.usedSectors.nextClearBit(RegionConstants.HEADER_SECTORS); + + while (candidate < this.totalSectors) { + int occupied = this.usedSectors.nextSetBit(candidate); + int available = occupied == -1 ? this.totalSectors - candidate : occupied - candidate; + + if (available >= count) { + break; + } + candidate = this.usedSectors.nextClearBit(candidate + available); + } + + this.usedSectors.set(candidate, candidate + count); + this.totalSectors = Math.max(this.totalSectors, candidate + count); + return candidate; + } + + /** + * Marks an existing range of sectors as used without searching for a free gap. + * The method is used while reading the header of an already existing region file. + * + * @param offset the index of the first sector of the range + * @param count the amount of sectors the range spans + * @throws IllegalArgumentException if the offset or the count is invalid + * @throws IllegalStateException if the range overlaps an already reserved range + */ + void reserve(int offset, int count) { + if (offset < RegionConstants.HEADER_SECTORS) { + throw new IllegalArgumentException("The sector offset must not point into the header but was " + offset); + } + if (count <= 0) { + throw new IllegalArgumentException("The sector count must be positive but was " + count); + } + int overlap = this.usedSectors.nextSetBit(offset); + + if (overlap != -1 && overlap < offset + count) { + throw new IllegalStateException("The sector range [" + offset + ", " + (offset + count) + ") overlaps sector " + overlap); + } + + this.usedSectors.set(offset, offset + count); + this.totalSectors = Math.max(this.totalSectors, offset + count); + } + + /** + * Marks a range of sectors as free so a later allocation can reuse it. + * The region file is never shrunk, the freed space stays part of the file. + * + * @param offset the index of the first sector of the range + * @param count the amount of sectors the range spans + */ + void free(int offset, int count) { + if (offset < RegionConstants.HEADER_SECTORS || count <= 0) { + return; + } + this.usedSectors.clear(offset, offset + count); + } + + /** + * Checks whether the given sector is currently not used by any chunk. + * + * @param sector the index of the sector to check + * @return true if the sector is free, otherwise false + */ + @Contract(pure = true) + boolean isFree(int sector) { + return !this.usedSectors.get(sector); + } + + /** + * Returns the amount of sectors the region file currently spans. + * + * @return the amount of sectors + */ + @Contract(pure = true) + int totalSectors() { + return this.totalSectors; + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/anvil/package-info.java b/src/main/java/net/theevilreaper/aves/instance/anvil/package-info.java new file mode 100644 index 00000000..fb458175 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/anvil/package-info.java @@ -0,0 +1,4 @@ +@NotNullByDefault +package net.theevilreaper.aves.instance.anvil; + +import org.jetbrains.annotations.NotNullByDefault; diff --git a/src/main/java/net/theevilreaper/aves/instance/light/BlockFace.java b/src/main/java/net/theevilreaper/aves/instance/light/BlockFace.java new file mode 100644 index 00000000..357f93ac --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/light/BlockFace.java @@ -0,0 +1,117 @@ +package net.theevilreaper.aves.instance.light; + +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; + +/** + * The {@link BlockFace} enum names the six faces of a block through which light can travel. + *

+ * The engine keeps its own enum instead of using the one of the server so the propagation can be + * verified without a running server. The offsets follow the block coordinate system, so a face + * describes both the side of a block and the direction a neighbour lies in. + *

+ *

+ * This type is experimental. The light engine is new and its API may still change. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public enum BlockFace { + + /** + * The face towards the negative y axis. + */ + BOTTOM(0, -1, 0), + + /** + * The face towards the positive y axis. + */ + TOP(0, 1, 0), + + /** + * The face towards the negative z axis. + */ + NORTH(0, 0, -1), + + /** + * The face towards the positive z axis. + */ + SOUTH(0, 0, 1), + + /** + * The face towards the negative x axis. + */ + WEST(-1, 0, 0), + + /** + * The face towards the positive x axis. + */ + EAST(1, 0, 0); + + private final int offsetX; + private final int offsetY; + private final int offsetZ; + + /** + * Creates a new face with the offset it points at. + * + * @param offsetX the offset on the x axis + * @param offsetY the offset on the y axis + * @param offsetZ the offset on the z axis + */ + BlockFace(int offsetX, int offsetY, int offsetZ) { + this.offsetX = offsetX; + this.offsetY = offsetY; + this.offsetZ = offsetZ; + } + + /** + * Returns the offset of this face on the x axis. + * + * @return the offset on the x axis + */ + @Contract(pure = true) + public int offsetX() { + return this.offsetX; + } + + /** + * Returns the offset of this face on the y axis. + * + * @return the offset on the y axis + */ + @Contract(pure = true) + public int offsetY() { + return this.offsetY; + } + + /** + * Returns the offset of this face on the z axis. + * + * @return the offset on the z axis + */ + @Contract(pure = true) + public int offsetZ() { + return this.offsetZ; + } + + /** + * Returns the face which points in the opposite direction. + * + * @return the opposite face + */ + @Contract(pure = true) + public BlockFace opposite() { + return switch (this) { + case BOTTOM -> TOP; + case TOP -> BOTTOM; + case NORTH -> SOUTH; + case SOUTH -> NORTH; + case WEST -> EAST; + case EAST -> WEST; + }; + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/light/BlockLightSource.java b/src/main/java/net/theevilreaper/aves/instance/light/BlockLightSource.java new file mode 100644 index 00000000..104019ce --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/light/BlockLightSource.java @@ -0,0 +1,42 @@ +package net.theevilreaper.aves.instance.light; + +import org.jetbrains.annotations.ApiStatus; + +/** + * The {@link BlockLightSource} interface describes the light properties of a block state. + *

+ * The propagation only needs to know two things about a block: how much light it emits and which of + * its faces light cannot pass. Keeping that behind an interface separates the algorithm from the + * registries of a running server, which is what allows the engine to be verified without starting + * one. It is the same separation the Anvil codec uses for its palette entries. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public interface BlockLightSource { + + /** + * Returns the amount of light the given block emits on its own. + * + * @param stateId the state id of the block + * @return the emitted light level between zero and {@link LightNibbles#MAX_LEVEL} + */ + int emission(int stateId); + + /** + * Checks whether light is unable to pass the given face of the block. + *

+ * The answer has to be given per face. Roughly one in seven block types of the game occludes + * some of its faces and not others, slabs, stairs, snow and farmland among them, so a single + * flag per block would answer this incorrectly for a large amount of real blocks. + *

+ * + * @param stateId the state id of the block + * @param face the face to check + * @return true if light cannot pass the face, otherwise false + */ + boolean blocksFace(int stateId, BlockFace face); +} diff --git a/src/main/java/net/theevilreaper/aves/instance/light/ChunkLightPropagator.java b/src/main/java/net/theevilreaper/aves/instance/light/ChunkLightPropagator.java new file mode 100644 index 00000000..c0f29e4f --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/light/ChunkLightPropagator.java @@ -0,0 +1,307 @@ +package net.theevilreaper.aves.instance.light; + +import org.jetbrains.annotations.ApiStatus; + +import java.util.ArrayList; +import java.util.List; + +/** + * The {@link ChunkLightPropagator} class spreads light through every section of a chunk at once. + *

+ * A propagation which stops at a section border produces a visible seam every sixteen blocks, + * because a light source near the border lights its own section and nothing beyond it. This class + * therefore treats the sections of a chunk as one column and lets the search cross their borders. + *

+ *

+ * The search is the same breadth-first pass {@link LightPropagator} performs, extended by the + * vertical axis. A position is visited again whenever a brighter source raises its level, so the + * queue can hold more entries than the column has positions and grows when it runs full. + *

+ *

+ * An instance keeps its buffers for the largest column it has seen and reuses them, so repeated + * runs allocate nothing beyond their result. It is reusable but confined to a single thread. + *

+ *

+ * This type is experimental. The light engine is new and its API may still change. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class ChunkLightPropagator { + + private static final BlockFace[] FACES = BlockFace.values(); + private static final int MASK = LightNibbles.DIMENSION - 1; + + private byte[] levels; + private int[] queue; + + /** + * Creates a new propagator without any buffer. The buffers are sized on the first run. + */ + public ChunkLightPropagator() { + this.levels = new byte[0]; + this.queue = new int[0]; + } + + /** + * Calculates the light of every section of a chunk. + * The sections are expected in the order they are stacked, starting with the lowest one. + * + * @param sections the light properties of every section of the chunk + * @return the calculated light of every section, in the order the sections were given + * @throws IllegalArgumentException if the chunk holds no section + */ + public List propagate(List sections) { + int height = prepare(sections); + return search(sections, height, seed(sections, height)); + } + + /** + * Calculates the sky light of every section of a chunk. + *

+ * Sky light enters from above and falls straight down without losing a level, which is what + * makes a cave dark while an open field is fully lit at every height. Only once something stops + * the fall does the light spread like any other light, losing one level per block. + *

+ * + * @param sections the light properties of every section of the chunk + * @return the calculated sky light of every section, in the order the sections were given + * @throws IllegalArgumentException if the chunk holds no section + */ + public List propagateSky(List sections) { + int height = prepare(sections); + return search(sections, height, seedSky(sections, height)); + } + + /** + * Verifies the given chunk and clears the buffers for a new run. + * + * @param sections the light properties of every section of the chunk + * @return the amount of blocks the column spans vertically + * @throws IllegalArgumentException if the chunk holds no section + */ + private int prepare(List sections) { + if (sections.isEmpty()) { + throw new IllegalArgumentException("A chunk has to hold at least one section"); + } + + int height = sections.size() * LightNibbles.DIMENSION; + ensureCapacity(height * LightNibbles.DIMENSION * LightNibbles.DIMENSION); + java.util.Arrays.fill(this.levels, 0, height * LightNibbles.DIMENSION * LightNibbles.DIMENSION, (byte) 0); + return height; + } + + /** + * Spreads the queued levels through the column. + * + * @param sections the light properties of every section of the chunk + * @param height the amount of blocks the column spans vertically + * @param queued the amount of positions which were queued as sources + * @return the calculated light of every section + */ + private List search(List sections, int height, int queued) { + int tail = queued; + int head = 0; + + while (head < tail) { + int index = this.queue[head++]; + int level = this.levels[index]; + + if (level <= 1) { + continue; + } + + int x = index & MASK; + int z = (index >> 4) & MASK; + int y = index >> 8; + int next = level - 1; + + for (BlockFace face : FACES) { + int neighbourX = x + face.offsetX(); + int neighbourY = y + face.offsetY(); + int neighbourZ = z + face.offsetZ(); + + if (isOutside(neighbourX, neighbourY, neighbourZ, height)) { + continue; + } + if (blocksFace(sections, neighbourX, neighbourY, neighbourZ, face.opposite())) { + continue; + } + + int neighbourIndex = index(neighbourX, neighbourY, neighbourZ); + + if (this.levels[neighbourIndex] >= next) { + continue; + } + this.levels[neighbourIndex] = (byte) next; + ensureRoom(tail); + this.queue[tail++] = neighbourIndex; + } + } + return collect(sections.size()); + } + + /** + * Makes room for one more entry in the queue. + *

+ * A position is queued again every time its level is raised, which happens when a brighter + * source reaches a position a dimmer one had already lit. The amount of entries is therefore + * not bounded by the amount of positions, and the queue has to be able to grow. + *

+ * + * @param tail the amount of entries the queue currently holds + */ + private void ensureRoom(int tail) { + if (tail == this.queue.length) { + this.queue = java.util.Arrays.copyOf(this.queue, this.queue.length * 2); + } + } + + /** + * Grows the buffers if the given column needs more room than the previous one. + * + * @param blockCount the amount of blocks the column holds + */ + private void ensureCapacity(int blockCount) { + if (this.levels.length < blockCount) { + this.levels = new byte[blockCount]; + this.queue = new int[blockCount]; + } + } + + /** + * Puts every emitting block of the column into the queue. + * + * @param sections the light properties of every section + * @param height the amount of blocks the column spans vertically + * @return the amount of queued positions + */ + private int seed(List sections, int height) { + int tail = 0; + + for (int y = 0; y < height; y++) { + SectionOpacity section = sections.get(y >> 4); + + if (!section.hasEmission()) { + continue; + } + int localY = y & MASK; + + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + int emission = section.emission(x, localY, z); + + if (emission <= 0) { + continue; + } + int index = index(x, y, z); + this.levels[index] = (byte) emission; + ensureRoom(tail); + this.queue[tail++] = index; + } + } + } + return tail; + } + + /** + * Puts every block which sees the open sky into the queue. + *

+ * Every column is walked from the top of the chunk downwards. As long as light can enter the + * block from above it receives the full level, which is why an open column is lit to the very + * bottom. The walk of a column ends at the first block that stops the light. + *

+ * + * @param sections the light properties of every section + * @param height the amount of blocks the column spans vertically + * @return the amount of queued positions + */ + private int seedSky(List sections, int height) { + int tail = 0; + + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + for (int y = height - 1; y >= 0; y--) { + if (blocksFace(sections, x, y, z, BlockFace.TOP)) { + break; + } + int index = index(x, y, z); + this.levels[index] = LightNibbles.MAX_LEVEL; + ensureRoom(tail); + this.queue[tail++] = index; + } + } + } + return tail; + } + + /** + * Checks whether light cannot enter the given position through the given face. + * + * @param sections the light properties of every section + * @param x the x coordinate inside the chunk + * @param y the y coordinate inside the column + * @param z the z coordinate inside the chunk + * @param face the face light would enter through + * @return true if light cannot pass the face, otherwise false + */ + private static boolean blocksFace(List sections, int x, int y, int z, BlockFace face) { + return sections.get(y >> 4).blocksFace(x, y & MASK, z, face); + } + + /** + * Transfers the calculated levels into one light section per section of the chunk. + * + * @param sectionCount the amount of sections the chunk holds + * @return the calculated light of every section + */ + private List collect(int sectionCount) { + List result = new ArrayList<>(sectionCount); + + for (int section = 0; section < sectionCount; section++) { + int base = section * LightNibbles.DIMENSION; + result.add(collectSection(base)); + } + return result; + } + + /** + * Transfers the levels of a single section into a light section. + * + * @param baseY the lowest y coordinate of the section inside the column + * @return the light of the section + */ + private LightNibbles collectSection(int baseY) { + // The levels of a section lie next to each other in the buffer of the whole column, because + // the index of a position puts its y coordinate into the highest bits. + return LightNibbles.ofLevels(this.levels, baseY << 8); + } + + /** + * Checks whether the given position lies outside of the column. + * + * @param x the x coordinate to check + * @param y the y coordinate to check + * @param z the z coordinate to check + * @param height the amount of blocks the column spans vertically + * @return true if the position is outside of the column, otherwise false + */ + private static boolean isOutside(int x, int y, int z, int height) { + return (x | y | z) < 0 || x >= LightNibbles.DIMENSION || z >= LightNibbles.DIMENSION || y >= height; + } + + /** + * Calculates the index of a block inside the column. + * + * @param x the x coordinate inside the chunk + * @param y the y coordinate inside the column + * @param z the z coordinate inside the chunk + * @return the index of the block + */ + private static int index(int x, int y, int z) { + return (y << 8) | (z << 4) | x; + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/light/ChunkLightService.java b/src/main/java/net/theevilreaper/aves/instance/light/ChunkLightService.java new file mode 100644 index 00000000..c5b1ccd7 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/light/ChunkLightService.java @@ -0,0 +1,385 @@ +package net.theevilreaper.aves.instance.light; + +import net.minestom.server.instance.Chunk; +import net.minestom.server.instance.Instance; +import net.minestom.server.instance.Section; +import net.minestom.server.instance.palette.Palette; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * The {@link ChunkLightService} class calculates the block light of a chunk and hands the result to + * the sections of that chunk. + *

+ * The service is the connection between the engine and a running server. It reads the block states + * of every section, runs the propagation and writes the result back through + * {@link net.minestom.server.instance.light.Light#set(byte[])}. That method belongs to the stable + * part of the light interface, which is why the service uses it instead of implementing the + * interface itself: the calculation methods of that interface are marked internal and their + * signatures may change between server versions. + *

+ *

+ * Because the result is handed over through the regular interface, the service works with any chunk + * of any loader, including chunks produced by the Anvil loader of Aves or the one of the server. + *

+ *

+ * Writing the light through {@code set} also clears the update flag of the section, so the server + * does not recompute what was just calculated. + *

+ *

+ * A single instance may be used by as many threads as one likes. The service holds nothing beyond + * the source it was built with, which only answers questions about a block and never changes, so + * every call brings its own working state and two calls cannot reach each other. That matters more + * here than elsewhere: a server lights the chunks around its players in parallel and keeps one + * service for the whole instance, and because {@code set} clears the update flag, a result which two + * threads had corrupted would never be recomputed. The world would simply carry wrong light. + *

+ *

+ * The working state of a call is the propagator, which keeps buffers and is therefore built per + * call rather than kept in a field. Its buffers are the entire cost of that choice, and an + * allocation per chunk is far cheaper than either handing every thread its own service or letting + * the threads take turns on a shared one, which would give up exactly the parallelism this service + * exists to allow. + *

+ *

+ * This type is experimental. The light engine is new and its API may still change. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class ChunkLightService { + + private static final Logger LOGGER = LoggerFactory.getLogger(ChunkLightService.class); + + private static final BlockFace[] HORIZONTAL_FACES = {BlockFace.NORTH, BlockFace.SOUTH, BlockFace.WEST, BlockFace.EAST}; + + /** + * The amount of chunks the exchanged area reaches beyond the chunk in the middle. + * A level of fifteen cannot survive more than one chunk of travel, so a further ring could not + * receive anything the middle chunk sends out. + */ + private static final int NEIGHBOURHOOD_RADIUS = 1; + + /** + * The edge length of the exchanged area in chunks. + */ + private static final int NEIGHBOURHOOD_SIZE = NEIGHBOURHOOD_RADIUS * 2 + 1; + + /** + * The amount of exchange rounds after which the exchange gives up. + * A level drops by one per chunk border at the very least, so fifteen rounds are enough for + * every reachable level and the cap only protects against a case which should not exist. + */ + private static final int MAX_EXCHANGE_ROUNDS = 16; + + private final BlockLightSource source; + + /** + * Creates a service which reads the block properties from the registry of the server. + */ + public ChunkLightService() { + this(new MinestomBlockLightSource()); + } + + /** + * Creates a service which reads the block properties from the given source. + * + * @param source the source which describes the light properties of a block + */ + public ChunkLightService(BlockLightSource source) { + this.source = source; + } + + /** + * Calculates the block light of the given chunk and stores it in its sections. + *

+ * The block states are read under the read lock of the chunk, the propagation runs without any + * lock, and only the transfer of the result is guarded again. The expensive part therefore + * never blocks another user of the chunk. + *

+ * + * @param chunk the chunk to light + */ + public void calculate(Chunk chunk) { + apply(chunk, new ChunkLightPropagator().propagate(opacityOf(chunk)), false); + } + + /** + * Builds the opacity table of every section of the given chunk. + * + * @param chunk the chunk to read + * @return the opacity table of every section + */ + private List opacityOf(Chunk chunk) { + List states = readStates(chunk); + List opacity = new ArrayList<>(states.size()); + + for (int[] section : states) { + opacity.add(SectionOpacity.of(section, this.source)); + } + return opacity; + } + + /** + * Writes the calculated light into the sections of the given chunk. + * + * @param chunk the chunk which receives the light + * @param light the calculated light of every section + * @param sky whether the sky light is written instead of the block light + */ + private static void apply(Chunk chunk, List light, boolean sky) { + chunk.lockWriteLock(); + try { + List
sections = chunk.getSections(); + + for (int index = 0; index < sections.size() && index < light.size(); index++) { + byte[] array = light.get(index).toDenseArray(); + Section section = sections.get(index); + + if (sky) { + section.skyLight().set(array); + continue; + } + section.blockLight().set(array); + } + } finally { + chunk.unlockWriteLock(); + } + } + + /** + * Calculates the sky light of the given chunk and stores it in its sections. + * + * @param chunk the chunk to light + */ + public void calculateSky(Chunk chunk) { + List opacity = opacityOf(chunk); + List light = new ChunkLightPropagator().propagateSky(opacity); + apply(chunk, light, true); + } + + /** + * Calculates the light of the given chunk and continues it into the chunks around it. + *

+ * A chunk which is lit on its own ends its light at the border, which shows up as a straight + * dark line every sixteen blocks. Handing the border to the direct neighbours once is not + * enough either, because light which enters a neighbour has to leave it again on another side + * to reach the chunk behind it, which is what a source in a corner does. + *

+ *

+ * The exchange therefore repeats over the whole area until no chunk of it raises a level any + * more. Every injection only ever raises levels, so the repetition walks towards a fixed point + * and reaches the same result regardless of the order the borders are handed over in. A cap on + * the amount of rounds keeps a case which should not exist from looping forever; hitting it is + * reported instead of silently accepted. + *

+ *

+ * Only chunks which the instance already holds take part. A neighbour which is not loaded is + * skipped rather than loaded, because lighting a chunk must not pull a world into memory. + *

+ * + * @param instance the instance which holds the chunk and its neighbours + * @param chunkX the chunk x coordinate + * @param chunkZ the chunk z coordinate + */ + public void calculateWithNeighbours(Instance instance, int chunkX, int chunkZ) { + if (instance.getChunk(chunkX, chunkZ) == null) { + return; + } + + @Nullable NeighbourhoodEntry[] neighbourhood = readNeighbourhood(instance, chunkX, chunkZ); + exchangeUntilSettled(neighbourhood, chunkX, chunkZ); + + for (@Nullable NeighbourhoodEntry entry : neighbourhood) { + if (entry == null) { + continue; + } + apply(entry.chunk(), entry.state().toSections(), false); + } + } + + /** + * Reads every already loaded chunk of the exchanged area and lights it on its own. + *

+ * The opacity table of a chunk is built here and nowhere else, because resolving the block + * states of a chunk is the expensive part of the whole operation and the exchange visits the + * same chunk many times. + *

+ * + * @param instance the instance which holds the chunks + * @param chunkX the chunk x coordinate of the middle of the area + * @param chunkZ the chunk z coordinate of the middle of the area + * @return one entry per position of the area, empty where no chunk is loaded + */ + private @Nullable NeighbourhoodEntry[] readNeighbourhood(Instance instance, int chunkX, int chunkZ) { + @Nullable NeighbourhoodEntry[] neighbourhood = new NeighbourhoodEntry[NEIGHBOURHOOD_SIZE * NEIGHBOURHOOD_SIZE]; + + for (int offsetZ = -NEIGHBOURHOOD_RADIUS; offsetZ <= NEIGHBOURHOOD_RADIUS; offsetZ++) { + for (int offsetX = -NEIGHBOURHOOD_RADIUS; offsetX <= NEIGHBOURHOOD_RADIUS; offsetX++) { + Chunk chunk = instance.getChunk(chunkX + offsetX, chunkZ + offsetZ); + + if (chunk == null) { + continue; + } + + List opacity = opacityOf(chunk); + neighbourhood[slot(offsetX, offsetZ)] = + new NeighbourhoodEntry(chunk, opacity, ChunkLightState.blockLight(opacity)); + } + } + return neighbourhood; + } + + /** + * Repeats the border exchange over the given area until nothing changes any more. + * + * @param neighbourhood the chunks of the exchanged area + * @param chunkX the chunk x coordinate of the middle of the area + * @param chunkZ the chunk z coordinate of the middle of the area + */ + private static void exchangeUntilSettled(@Nullable NeighbourhoodEntry[] neighbourhood, int chunkX, int chunkZ) { + for (int round = 0; round < MAX_EXCHANGE_ROUNDS; round++) { + if (!exchange(neighbourhood)) { + return; + } + } + LOGGER.warn( + "The light of a chunk and its neighbours did not settle after {} exchange rounds chunk=[{},{}]", + MAX_EXCHANGE_ROUNDS, chunkX, chunkZ + ); + } + + /** + * Hands the border of every loaded neighbour to every loaded chunk of the area once. + *

+ * The area is walked in a fixed order so that two runs over the same chunks do the same work + * in the same sequence. + *

+ * + * @param neighbourhood the chunks of the exchanged area + * @return true if at least one chunk raised a level, otherwise false + */ + private static boolean exchange(@Nullable NeighbourhoodEntry[] neighbourhood) { + boolean changed = false; + + for (int offsetZ = -NEIGHBOURHOOD_RADIUS; offsetZ <= NEIGHBOURHOOD_RADIUS; offsetZ++) { + for (int offsetX = -NEIGHBOURHOOD_RADIUS; offsetX <= NEIGHBOURHOOD_RADIUS; offsetX++) { + @Nullable NeighbourhoodEntry entry = neighbourhood[slot(offsetX, offsetZ)]; + + if (entry == null) { + continue; + } + + for (BlockFace face : HORIZONTAL_FACES) { + int neighbourX = offsetX + face.offsetX(); + int neighbourZ = offsetZ + face.offsetZ(); + + if (isOutside(neighbourX, neighbourZ)) { + continue; + } + + @Nullable NeighbourhoodEntry neighbour = neighbourhood[slot(neighbourX, neighbourZ)]; + + if (neighbour == null) { + continue; + } + changed |= entry.state().injectBorder( + entry.opacity(), face, neighbour.state().border(face.opposite()) + ); + } + } + } + return changed; + } + + /** + * Calculates the position of a chunk inside the exchanged area. + * + * @param offsetX the chunk x offset from the middle of the area + * @param offsetZ the chunk z offset from the middle of the area + * @return the position of the chunk inside the area + */ + @Contract(pure = true) + private static int slot(int offsetX, int offsetZ) { + return (offsetZ + NEIGHBOURHOOD_RADIUS) * NEIGHBOURHOOD_SIZE + (offsetX + NEIGHBOURHOOD_RADIUS); + } + + /** + * Checks whether the given offset lies outside of the exchanged area. + * + * @param offsetX the chunk x offset from the middle of the area + * @param offsetZ the chunk z offset from the middle of the area + * @return true if the offset is outside of the area, otherwise false + */ + @Contract(pure = true) + private static boolean isOutside(int offsetX, int offsetZ) { + return Math.abs(offsetX) > NEIGHBOURHOOD_RADIUS || Math.abs(offsetZ) > NEIGHBOURHOOD_RADIUS; + } + + /** + * The {@link NeighbourhoodEntry} record holds everything the exchange needs about one chunk of + * the area, so neither its block states nor its opacity tables are read a second time. + * + * @param chunk the chunk the entry belongs to + * @param opacity the light properties of every section of the chunk + * @param state the light of the chunk as it is exchanged + */ + private record NeighbourhoodEntry( + Chunk chunk, + List opacity, + ChunkLightState state + ) { + } + + /** + * Returns the block light level which is stored for the given position. + * + * @param chunk the chunk which holds the position + * @param x the x coordinate inside the chunk + * @param y the y coordinate of the block + * @param z the z coordinate inside the chunk + * @return the stored light level of the position + */ + @Contract(pure = true) + public int blockLightAt(Chunk chunk, int x, int y, int z) { + chunk.lockReadLock(); + try { + return chunk.getSectionAt(y).blockLight().getLevel(x & 15, y & 15, z & 15); + } finally { + chunk.unlockReadLock(); + } + } + + /** + * Reads the block state of every block of every section of the chunk. + * + * @param chunk the chunk to read + * @return the state ids of every section, ordered from the lowest section upwards + */ + private static List readStates(Chunk chunk) { + chunk.lockReadLock(); + try { + List
sections = chunk.getSections(); + List states = new ArrayList<>(sections.size()); + + for (Section section : sections) { + int[] blocks = new int[LightNibbles.BLOCK_COUNT]; + Palette palette = section.blockPalette(); + palette.getAll((x, y, z, value) -> blocks[(y << 8) | (z << 4) | x] = value); + states.add(blocks); + } + return states; + } finally { + chunk.unlockReadLock(); + } + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/light/ChunkLightState.java b/src/main/java/net/theevilreaper/aves/instance/light/ChunkLightState.java new file mode 100644 index 00000000..e8ba6882 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/light/ChunkLightState.java @@ -0,0 +1,674 @@ +package net.theevilreaper.aves.instance.light; + +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; + +import java.util.ArrayList; +import java.util.List; + +/** + * The {@link ChunkLightState} class holds the calculated light of a chunk and updates it when a + * single block changed, without recalculating the whole chunk. + *

+ * Adding brightness is easy: the new light spreads and never has to take anything back. Removing it + * is the hard case and the reason this class exists. When a light source disappears, the brightness + * it had spread is still stored in every block around it, and simply spreading again would keep + * that glow forever. The update therefore runs in two passes: the first retracts every level which + * originated from the changed position, collecting the still valid levels it meets at the edge, and + * the second spreads those back in. + *

+ *

+ * Sky light needs one more piece of state for the same reason. Its origin is not a block but the + * open sky above a column, so an update cannot tell from the levels alone which positions lost + * their origin and which gained one. A state which holds sky light therefore keeps the height at + * which every column stops the sky and compares it against the height the column has after the + * change. Only that one column can move, which is what keeps an update small instead of re-seeding + * all two hundred and fifty six columns of the chunk. + *

+ *

+ * Instances are not thread safe. Keep one per chunk and use it from one thread at a time. + *

+ *

+ * This type is experimental. The light engine is new and its API may still change. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class ChunkLightState { + + private static final BlockFace[] FACES = BlockFace.values(); + private static final int MASK = LightNibbles.DIMENSION - 1; + + /** + * The heightmap of a state which holds block light. Block light never reads it. + */ + private static final int[] NO_HEIGHTMAP = new int[0]; + + /** + * The height of a column which nothing stops the sky in. + */ + private static final int OPEN_COLUMN = -1; + + private final byte[] levels; + private final int sectionCount; + private final int height; + private final boolean sky; + private final int[] skyTop; + + private final int[] removalQueue; + private final byte[] removalLevels; + private int[] additionQueue; + + /** + * Creates a new state from the given levels. + * + * @param levels the level of every block of the column + * @param sectionCount the amount of sections the chunk holds + * @param sky whether the state holds sky light + * @param skyTop the highest position which stops the sky per column + */ + private ChunkLightState(byte[] levels, int sectionCount, boolean sky, int[] skyTop) { + this.levels = levels; + this.sectionCount = sectionCount; + this.height = sectionCount * LightNibbles.DIMENSION; + this.sky = sky; + this.skyTop = skyTop; + this.removalQueue = new int[levels.length]; + this.removalLevels = new byte[levels.length]; + this.additionQueue = new int[levels.length]; + } + + + /** + * Makes room for one more entry in the addition queue. + *

+ * A position enters the queue again whenever a brighter source raises it, and the retraction can + * reach the same position from several sides. The amount of entries is therefore not bounded by + * the amount of positions, so the queue has to be able to grow rather than rely on that + * assumption. + *

+ * + * @param tail the amount of entries the queue currently holds + */ + private void ensureRoom(int tail) { + if (tail == this.additionQueue.length) { + this.additionQueue = java.util.Arrays.copyOf(this.additionQueue, this.additionQueue.length * 2); + } + } + + /** + * Calculates the block light of a chunk and keeps it for later updates. + * + * @param sections the light properties of every section of the chunk + * @return the created state + */ + public static ChunkLightState blockLight(List sections) { + return of(sections, new ChunkLightPropagator().propagate(sections), false); + } + + /** + * Calculates the sky light of a chunk and keeps it for later updates. + * + * @param sections the light properties of every section of the chunk + * @return the created state + */ + public static ChunkLightState skyLight(List sections) { + return of(sections, new ChunkLightPropagator().propagateSky(sections), true); + } + + /** + * Builds a state from an already calculated light. + * + * @param sections the light properties of every section of the chunk + * @param light the calculated light of every section + * @param sky whether the light is sky light + * @return the created state + */ + private static ChunkLightState of(List sections, List light, boolean sky) { + int sectionCount = sections.size(); + byte[] levels = new byte[sectionCount * LightNibbles.BLOCK_COUNT]; + + for (int section = 0; section < sectionCount; section++) { + LightNibbles nibbles = light.get(section); + int base = section * LightNibbles.DIMENSION; + + for (int y = 0; y < LightNibbles.DIMENSION; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + levels[index(x, base + y, z)] = (byte) nibbles.get(x, y, z); + } + } + } + } + return new ChunkLightState(levels, sectionCount, sky, sky ? skyTopOf(sections) : NO_HEIGHTMAP); + } + + /** + * Determines for every column of the chunk where the sky stops. + * + * @param sections the light properties of every section of the chunk + * @return the highest position which stops the sky per column + */ + @Contract(pure = true) + private static int[] skyTopOf(List sections) { + int height = sections.size() * LightNibbles.DIMENSION; + int[] skyTop = new int[LightNibbles.DIMENSION * LightNibbles.DIMENSION]; + + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + skyTop[column(x, z)] = columnTop(sections, height, x, z); + } + } + return skyTop; + } + + /** + * Determines where the sky stops in a single column. + *

+ * The walk starts at the top of the chunk and ends at the first block which light cannot enter + * from above, exactly as the initial sky propagation walks a column. Everything above that + * block sees the open sky, everything below it does not. + *

+ * + * @param sections the light properties of every section of the chunk + * @param height the amount of blocks the column spans vertically + * @param x the x coordinate inside the chunk + * @param z the z coordinate inside the chunk + * @return the highest position which stops the sky, or a negative value for an open column + */ + @Contract(pure = true) + private static int columnTop(List sections, int height, int x, int z) { + for (int y = height - 1; y >= 0; y--) { + if (blocksFace(sections, x, y, z, BlockFace.TOP)) { + return y; + } + } + return OPEN_COLUMN; + } + + /** + * Returns the level which is currently stored for the given position. + * + * @param x the x coordinate inside the chunk + * @param y the y coordinate inside the column + * @param z the z coordinate inside the chunk + * @return the stored level of the position + */ + @Contract(pure = true) + public int get(int x, int y, int z) { + return this.levels[index(x, y, z)]; + } + + /** + * Updates the light after the block at the given position changed. + * + * @param sections the light properties of every section, reflecting the change + * @param x the x coordinate inside the chunk + * @param y the y coordinate inside the column + * @param z the z coordinate inside the chunk + */ + public void update(List sections, int x, int y, int z) { + if (this.sky) { + updateSky(sections, x, y, z); + return; + } + + int additions = retract(seedRemoval(0, index(x, y, z))); + additions = seedEmission(sections, additions); + spread(sections, additions); + } + + /** + * Updates the sky light after the block at the given position changed. + *

+ * Only the column of the changed block can stop the sky at another height than before, so only + * that column is walked again. The difference between the old and the new height names exactly + * the positions which changed their origin: the ones that fell out of the open sky have to give + * their level back, the ones that fell into it receive the full level. + *

+ *

+ * The changed position itself is retracted in either case, because it carries the light of the + * block that is gone. Its neighbours are handed to the second pass afterwards, since a position + * which just turned transparent holds no light of its own that a retraction could follow and + * has to be filled from the outside instead. + *

+ * + * @param sections the light properties of every section, reflecting the change + * @param x the x coordinate inside the chunk + * @param y the y coordinate inside the column + * @param z the z coordinate inside the chunk + */ + private void updateSky(List sections, int x, int y, int z) { + int column = column(x, z); + int previousTop = this.skyTop[column]; + int currentTop = columnTop(sections, this.height, x, z); + this.skyTop[column] = currentTop; + + int removals = seedRemoval(0, index(x, y, z)); + + for (int lost = previousTop + 1; lost <= currentTop; lost++) { + if (lost != y) { + removals = seedRemoval(removals, index(x, lost, z)); + } + } + + int additions = retract(removals); + + for (int opened = currentTop + 1; opened <= previousTop; opened++) { + additions = seedSky(additions, index(x, opened, z)); + } + + // A position above both heights kept its open sky and only lost its level to the retraction. + if (y > currentTop && y > previousTop) { + additions = seedSky(additions, index(x, y, z)); + } + spread(sections, seedNeighbours(additions, x, y, z)); + } + + /** + * Clears the level of a position and hands it to the retraction. + * + * @param queued the amount of positions which are already queued for the retraction + * @param index the index of the position to retract + * @return the amount of queued positions + */ + private int seedRemoval(int queued, int index) { + this.removalQueue[queued] = index; + this.removalLevels[queued] = this.levels[index]; + this.levels[index] = 0; + return queued + 1; + } + + /** + * Gives a position which sees the open sky its full level and hands it to the second pass. + * + * @param queued the amount of positions which are already queued for the second pass + * @param index the index of the position which sees the sky + * @return the amount of queued positions + */ + private int seedSky(int queued, int index) { + this.levels[index] = LightNibbles.MAX_LEVEL; + this.additionQueue[queued] = index; + return queued + 1; + } + + /** + * Hands every neighbour of the changed position which still carries light to the second pass. + * + * @param queued the amount of positions which are already queued for the second pass + * @param x the x coordinate inside the chunk + * @param y the y coordinate inside the column + * @param z the z coordinate inside the chunk + * @return the amount of queued positions + */ + private int seedNeighbours(int queued, int x, int y, int z) { + int tail = queued; + + for (BlockFace face : FACES) { + int neighbourX = x + face.offsetX(); + int neighbourY = y + face.offsetY(); + int neighbourZ = z + face.offsetZ(); + + if (isOutside(neighbourX, neighbourY, neighbourZ)) { + continue; + } + + int neighbourIndex = index(neighbourX, neighbourY, neighbourZ); + + if (this.levels[neighbourIndex] <= 1) { + continue; + } + ensureRoom(tail); + this.additionQueue[tail++] = neighbourIndex; + } + return tail; + } + + /** + * Retracts every level which originated from the already cleared positions. + *

+ * A neighbour which is darker than the level being removed can only have received its light + * from it, so it is cleared as well. A neighbour which is as bright or brighter has another + * origin and becomes a starting point for the second pass instead. Every position the caller + * seeded is cleared before the walk begins, so a seed never mistakes another seed for a source + * that is still valid. + *

+ * + * @param seeded the amount of positions the caller handed to the retraction + * @return the amount of positions which were queued for the second pass + */ + private int retract(int seeded) { + int removalTail = seeded; + int additionTail = 0; + + for (int head = 0; head < removalTail; head++) { + int index = this.removalQueue[head]; + int removed = this.removalLevels[head]; + + if (removed == 0) { + continue; + } + + int x = index & MASK; + int z = (index >> 4) & MASK; + int y = index >> 8; + + for (BlockFace face : FACES) { + int neighbourX = x + face.offsetX(); + int neighbourY = y + face.offsetY(); + int neighbourZ = z + face.offsetZ(); + + if (isOutside(neighbourX, neighbourY, neighbourZ)) { + continue; + } + + int neighbourIndex = index(neighbourX, neighbourY, neighbourZ); + int level = this.levels[neighbourIndex]; + + if (level == 0) { + continue; + } + if (level < removed) { + this.removalQueue[removalTail] = neighbourIndex; + this.removalLevels[removalTail++] = (byte) level; + this.levels[neighbourIndex] = 0; + continue; + } + ensureRoom(additionTail); + this.additionQueue[additionTail++] = neighbourIndex; + } + } + return additionTail; + } + + /** + * Adds every position which produces light on its own to the second pass. + *

+ * A block which turns transparent holds no light a retraction could follow, so the sources of + * the chunk are offered again and refill the position that opened up. + *

+ * + * @param sections the light properties of every section + * @param queued the amount of positions which are already queued + * @return the amount of queued positions + */ + private int seedEmission(List sections, int queued) { + int tail = queued; + + for (int y = 0; y < this.height; y++) { + SectionOpacity section = sections.get(y >> 4); + + if (!section.hasEmission()) { + continue; + } + int localY = y & MASK; + + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + int emission = section.emission(x, localY, z); + + if (emission <= 0) { + continue; + } + int index = index(x, y, z); + + if (this.levels[index] < emission) { + this.levels[index] = (byte) emission; + } + ensureRoom(tail); + this.additionQueue[tail++] = index; + } + } + } + return tail; + } + + /** + * Spreads the queued levels back into the retracted area. + * + * @param sections the light properties of every section + * @param queued the amount of queued positions + */ + private void spread(List sections, int queued) { + int tail = queued; + + for (int head = 0; head < tail; head++) { + int index = this.additionQueue[head]; + int level = this.levels[index]; + + if (level <= 1) { + continue; + } + + int x = index & MASK; + int z = (index >> 4) & MASK; + int y = index >> 8; + int next = level - 1; + + for (BlockFace face : FACES) { + int neighbourX = x + face.offsetX(); + int neighbourY = y + face.offsetY(); + int neighbourZ = z + face.offsetZ(); + + if (isOutside(neighbourX, neighbourY, neighbourZ)) { + continue; + } + if (blocksFace(sections, neighbourX, neighbourY, neighbourZ, face.opposite())) { + continue; + } + + int neighbourIndex = index(neighbourX, neighbourY, neighbourZ); + + if (this.levels[neighbourIndex] >= next) { + continue; + } + this.levels[neighbourIndex] = (byte) next; + ensureRoom(tail); + this.additionQueue[tail++] = neighbourIndex; + } + } + } + + /** + * Returns the light levels along one horizontal border of the chunk. + *

+ * The result is read by the neighbouring chunk to continue the light across the border. It is + * ordered by height first and by the remaining horizontal axis second. + *

+ * + * @param face the border to read + * @return the level of every block along the border + * @throws IllegalArgumentException if the given face is not horizontal + */ + @Contract(pure = true) + public byte[] border(BlockFace face) { + checkHorizontal(face); + byte[] border = new byte[this.height * LightNibbles.DIMENSION]; + + for (int y = 0; y < this.height; y++) { + for (int offset = 0; offset < LightNibbles.DIMENSION; offset++) { + border[y * LightNibbles.DIMENSION + offset] = this.levels[borderIndex(face, y, offset)]; + } + } + return border; + } + + /** + * Feeds the light of a neighbouring chunk into this one. + *

+ * Without this a light source close to the edge of a chunk lights its own chunk and stops + * abruptly at the border, which shows up as a straight dark line every sixteen blocks. Each + * level of the neighbour arrives one level weaker, exactly as if the two chunks had been + * calculated together. + *

+ * + *

+ * The answer tells the caller whether the injection raised anything at all. An exchange over + * several chunks repeats until every one of them reports that nothing changed, which is the + * point at which the light of the whole area is settled. + *

+ * + * @param sections the light properties of every section of this chunk + * @param face the border the light enters through + * @param border the levels along the matching border of the neighbour + * @return true if at least one level of this chunk was raised, otherwise false + * @throws IllegalArgumentException if the face is not horizontal or the border has the wrong size + */ + public boolean injectBorder(List sections, BlockFace face, byte[] border) { + checkHorizontal(face); + + if (border.length != this.height * LightNibbles.DIMENSION) { + throw new IllegalArgumentException( + "The border of this chunk holds " + (this.height * LightNibbles.DIMENSION) + + " levels but the given one holds " + border.length + ); + } + + int tail = 0; + + for (int y = 0; y < this.height; y++) { + for (int offset = 0; offset < LightNibbles.DIMENSION; offset++) { + int incoming = border[y * LightNibbles.DIMENSION + offset] - 1; + + if (incoming <= 0) { + continue; + } + + int index = borderIndex(face, y, offset); + + if (this.levels[index] >= incoming) { + continue; + } + // The light enters through the face that lies towards the neighbour. + if (blocksFace(sections, index & MASK, index >> 8, (index >> 4) & MASK, face)) { + continue; + } + this.levels[index] = (byte) incoming; + ensureRoom(tail); + this.additionQueue[tail++] = index; + } + } + spread(sections, tail); + return tail > 0; + } + + /** + * Calculates the index of a block which lies on the given border. + * + * @param face the border the block lies on + * @param y the y coordinate inside the column + * @param offset the position along the remaining horizontal axis + * @return the index of the block + */ + @Contract(pure = true) + private static int borderIndex(BlockFace face, int y, int offset) { + return switch (face) { + case WEST -> index(0, y, offset); + case EAST -> index(LightNibbles.DIMENSION - 1, y, offset); + case NORTH -> index(offset, y, 0); + case SOUTH -> index(offset, y, LightNibbles.DIMENSION - 1); + default -> throw new IllegalArgumentException("The face " + face + " is not horizontal"); + }; + } + + /** + * Verifies that the given face describes a horizontal border. + * + * @param face the face to check + * @throws IllegalArgumentException if the face is not horizontal + */ + private static void checkHorizontal(BlockFace face) { + if (face == BlockFace.TOP || face == BlockFace.BOTTOM) { + throw new IllegalArgumentException( + "Only a horizontal border is shared between two chunks but " + face + " was given" + ); + } + } + + /** + * Returns the stored light as one light section per section of the chunk. + * + * @return the light of every section + */ + @Contract(pure = true) + public List toSections() { + List result = new ArrayList<>(this.sectionCount); + + for (int section = 0; section < this.sectionCount; section++) { + int base = section * LightNibbles.DIMENSION; + LightNibbles nibbles = LightNibbles.uniform(0); + boolean uniform = true; + int first = this.levels[index(0, base, 0)]; + + for (int y = 0; y < LightNibbles.DIMENSION; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + int level = this.levels[index(x, base + y, z)]; + + if (level != first) { + uniform = false; + } + if (level != 0) { + nibbles.set(x, y, z, level); + } + } + } + } + result.add(uniform ? LightNibbles.uniform(first) : nibbles); + } + return result; + } + + /** + * Checks whether light cannot enter the given position through the given face. + * + * @param sections the light properties of every section + * @param x the x coordinate inside the chunk + * @param y the y coordinate inside the column + * @param z the z coordinate inside the chunk + * @param face the face light would enter through + * @return true if light cannot pass the face, otherwise false + */ + private static boolean blocksFace(List sections, int x, int y, int z, BlockFace face) { + return sections.get(y >> 4).blocksFace(x, y & MASK, z, face); + } + + /** + * Checks whether the given position lies outside of the column. + * + * @param x the x coordinate to check + * @param y the y coordinate to check + * @param z the z coordinate to check + * @return true if the position is outside of the column, otherwise false + */ + private boolean isOutside(int x, int y, int z) { + return (x | y | z) < 0 || x >= LightNibbles.DIMENSION || z >= LightNibbles.DIMENSION || y >= this.height; + } + + /** + * Calculates the index of a block inside the column. + * + * @param x the x coordinate inside the chunk + * @param y the y coordinate inside the column + * @param z the z coordinate inside the chunk + * @return the index of the block + */ + private static int index(int x, int y, int z) { + return (y << 8) | (z << 4) | x; + } + + /** + * Calculates the index of a column of the chunk. + * + * @param x the x coordinate inside the chunk + * @param z the z coordinate inside the chunk + * @return the index of the column + */ + @Contract(pure = true) + private static int column(int x, int z) { + return (z << 4) | x; + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/light/LightNibbles.java b/src/main/java/net/theevilreaper/aves/instance/light/LightNibbles.java new file mode 100644 index 00000000..0da6d80c --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/light/LightNibbles.java @@ -0,0 +1,340 @@ +package net.theevilreaper.aves.instance.light; + +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.Nullable; + +import java.util.Arrays; + +/** + * The {@link LightNibbles} class stores the light level of every block of a section. + *

+ * A level occupies four bits, so two of them share a byte and a full section needs + * {@value #ARRAY_LENGTH} bytes. A section in which every block holds the same level is kept without + * an array at all, which is the common case: most sections of a world are either completely dark or + * completely lit by the sky. The array is allocated the moment a level differs from the rest and is + * released again when the section becomes uniform. + *

+ *

+ * Instances are not thread safe. A propagation builds them on one thread and publishes the result + * afterwards. + *

+ *

+ * This type is experimental. The light engine is new and its API may still change. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class LightNibbles { + + /** + * The edge length of a section in blocks. + */ + public static final int DIMENSION = 16; + + /** + * The amount of blocks a section holds. + */ + public static final int BLOCK_COUNT = DIMENSION * DIMENSION * DIMENSION; + + /** + * The amount of bytes a fully stored section occupies. + */ + public static final int ARRAY_LENGTH = BLOCK_COUNT / 2; + + /** + * The highest light level a block can carry. + */ + public static final int MAX_LEVEL = 15; + + private byte @Nullable [] levels; + private int uniformLevel; + + /** + * Creates a section which stores the given level for every block. + * + * @param level the level every block of the section carries + */ + private LightNibbles(int level) { + this.levels = null; + this.uniformLevel = level; + } + + /** + * Creates a section in which every block carries the given level. + * No array is allocated for it. + * + * @param level the level every block of the section carries + * @return the created section + * @throws IllegalArgumentException if the level is outside of the allowed range + */ + @Contract(pure = true, value = "_ -> new") + public static LightNibbles uniform(int level) { + return new LightNibbles(checkLevel(level)); + } + + /** + * Creates a section from a stored array. + * An array which holds a single repeated level is collapsed instead of being kept. + * + * @param array the stored bytes of the section + * @return the created section + * @throws IllegalArgumentException if the array does not have the expected length + */ + @Contract(pure = true, value = "_ -> new") + public static LightNibbles of(byte[] array) { + if (array.length != ARRAY_LENGTH) { + throw new IllegalArgumentException( + "A light section holds " + ARRAY_LENGTH + " bytes but the given array holds " + array.length + ); + } + + int uniform = uniformLevelOf(array); + + if (uniform >= 0) { + return new LightNibbles(uniform); + } + + LightNibbles nibbles = new LightNibbles(0); + nibbles.levels = array.clone(); + return nibbles; + } + + /** + * Creates a section from one level per position, packing two of them into a byte. + *

+ * A propagation calculates its levels into a flat array of one byte per position and has to + * hand them over in the packed form. Writing that result back through {@link #set(int, int, int, int)} + * costs a call, a range check and a read of the byte the level shares with its neighbour for + * every single position. Packing the two neighbours together instead reads every level once and + * writes every byte once, and it is the same operation for all 4096 of them. + *

+ *

+ * The levels of a section lie next to each other in the index order of the section, so a whole + * chunk column can keep the levels of all of its sections in one array and hand out one section + * of it through the offset. + *

+ * + * @param levels one level per position, starting at the given offset + * @param offset the position at which the section starts inside the array + * @return the created section + * @throws IllegalArgumentException if the array does not hold a whole section behind the offset + * or if a level is outside of the allowed range + */ + @Contract(pure = true, value = "_, _ -> new") + static LightNibbles ofLevels(byte[] levels, int offset) { + if (offset < 0 || levels.length - offset < BLOCK_COUNT) { + throw new IllegalArgumentException( + "A section holds " + BLOCK_COUNT + " levels but the given array holds " + + (levels.length - offset) + " behind the offset " + offset + ); + } + + byte[] packed = new byte[ARRAY_LENGTH]; + int first = levels[offset]; + int differing = 0; + int outOfRange = 0; + + for (int index = 0; index < ARRAY_LENGTH; index++) { + int low = levels[offset + (index << 1)]; + int high = levels[offset + (index << 1) + 1]; + packed[index] = (byte) ((low & 0x0F) | ((high & 0x0F) << 4)); + differing |= (low ^ first) | (high ^ first); + outOfRange |= low | high; + } + + // Every level is checked at once rather than one at a time. A level outside of a nibble + // sets a bit no level may reach, and one accumulated value carries the bits of all of them. + if ((outOfRange & ~0x0F) != 0) { + throw new IllegalArgumentException("A light level must be within [0, " + MAX_LEVEL + "]"); + } + + if (differing == 0) { + return new LightNibbles(first); + } + + LightNibbles nibbles = new LightNibbles(0); + nibbles.levels = packed; + return nibbles; + } + + /** + * Returns the light level of the given block. + * + * @param x the x coordinate inside the section + * @param y the y coordinate inside the section + * @param z the z coordinate inside the section + * @return the level of the block + */ + @Contract(pure = true) + public int get(int x, int y, int z) { + byte[] array = this.levels; + + if (array == null) { + return this.uniformLevel; + } + + int index = index(x, y, z); + return (array[index >> 1] >> ((index & 1) << 2)) & 0x0F; + } + + /** + * Sets the light level of the given block. + * Writing the level the section already holds everywhere keeps it without an array. + * + * @param x the x coordinate inside the section + * @param y the y coordinate inside the section + * @param z the z coordinate inside the section + * @param level the level to store + * @throws IllegalArgumentException if the level is outside of the allowed range + */ + public void set(int x, int y, int z, int level) { + checkLevel(level); + + if (this.levels == null) { + if (level == this.uniformLevel) { + return; + } + this.levels = expand(this.uniformLevel); + } + + int index = index(x, y, z); + int shift = (index & 1) << 2; + int position = index >> 1; + this.levels[position] = (byte) ((this.levels[position] & (0xF0 >>> shift)) | (level << shift)); + } + + /** + * Sets the level of every block of the section and releases the array. + * + * @param level the level every block of the section carries afterwards + * @throws IllegalArgumentException if the level is outside of the allowed range + */ + public void fill(int level) { + this.uniformLevel = checkLevel(level); + this.levels = null; + } + + /** + * Checks whether every block of the section carries the same level. + * + * @return true if the section holds a single level, otherwise false + */ + @Contract(pure = true) + public boolean isUniform() { + return this.levels == null; + } + + /** + * Returns the bytes of the section as they are stored. + * A uniform section of level zero reports an empty array, which is how the format stores a + * section without any light. Every other section reports its full bytes. + * + * @return a copy of the stored bytes + */ + @Contract(pure = true) + public byte[] toArray() { + byte[] array = this.levels; + + if (array != null) { + return array.clone(); + } + return this.uniformLevel == 0 ? new byte[0] : expand(this.uniformLevel); + } + + /** + * Returns the bytes of the section, expanding a uniform section into a full array. + * + * @return a copy of the bytes of every block + */ + @Contract(pure = true) + public byte[] toDenseArray() { + byte[] array = this.levels; + return array != null ? array.clone() : expand(this.uniformLevel); + } + + /** + * Creates a section which holds the same levels but shares no storage with this one. + * + * @return the created copy + */ + @Contract(pure = true, value = "-> new") + public LightNibbles copy() { + LightNibbles copy = new LightNibbles(this.uniformLevel); + byte[] array = this.levels; + + if (array != null) { + copy.levels = array.clone(); + } + return copy; + } + + /** + * Calculates the index of a block inside the section. + * + * @param x the x coordinate inside the section + * @param y the y coordinate inside the section + * @param z the z coordinate inside the section + * @return the index of the block + */ + @Contract(pure = true) + private static int index(int x, int y, int z) { + return (y << 8) | (z << 4) | x; + } + + /** + * Builds a full array in which every block carries the given level. + * + * @param level the level every block carries + * @return the created array + */ + @Contract(pure = true) + private static byte[] expand(int level) { + byte[] array = new byte[ARRAY_LENGTH]; + + if (level != 0) { + Arrays.fill(array, (byte) (level | (level << 4))); + } + return array; + } + + /** + * Determines whether the given array holds a single repeated level. + * + * @param array the array to inspect + * @return the repeated level or a negative value if the array holds more than one + */ + @Contract(pure = true) + private static int uniformLevelOf(byte[] array) { + byte first = array[0]; + int low = first & 0x0F; + + if (low != ((first >> 4) & 0x0F)) { + return -1; + } + for (byte value : array) { + if (value != first) { + return -1; + } + } + return low; + } + + /** + * Verifies that the given level can be stored in a nibble. + * + * @param level the level to check + * @return the given level + * @throws IllegalArgumentException if the level is outside of the allowed range + */ + @Contract(pure = true) + private static int checkLevel(int level) { + if (level < 0 || level > MAX_LEVEL) { + throw new IllegalArgumentException("A light level must be within [0, " + MAX_LEVEL + "] but was " + level); + } + return level; + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/light/LightPropagator.java b/src/main/java/net/theevilreaper/aves/instance/light/LightPropagator.java new file mode 100644 index 00000000..d43ff111 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/light/LightPropagator.java @@ -0,0 +1,177 @@ +package net.theevilreaper.aves.instance.light; + +import org.jetbrains.annotations.ApiStatus; + +import java.util.Arrays; + +/** + * The {@link LightPropagator} class spreads the light of every emitting block through a section. + *

+ * The propagation is a breadth-first search. A position is queued again whenever a brighter source + * raises its level, which happens when sources of different brightness reach the same area. The + * queue therefore holds more entries than the section has positions and grows when it runs full. + *

+ *

+ * An instance keeps its working buffers and reuses them across runs, which is what makes repeated + * propagation allocation free apart from the result. The buffers are cleared at the start of every + * run, so results never bleed from one run into the next. An instance is therefore reusable but + * confined to a single thread; use one per worker. + *

+ *

+ * This type is experimental. The light engine is new and its API may still change. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class LightPropagator { + + private static final BlockFace[] FACES = BlockFace.values(); + private static final int MASK = LightNibbles.DIMENSION - 1; + + private final byte[] levels; + private int[] queue; + + /** + * Creates a new propagator with its own working buffers. + */ + public LightPropagator() { + this.levels = new byte[LightNibbles.BLOCK_COUNT]; + this.queue = new int[LightNibbles.BLOCK_COUNT]; + } + + /** + * Makes room for one more entry in the queue. + *

+ * A position is queued again every time its level is raised, which happens when a brighter + * source reaches a position a dimmer one had already lit. The amount of entries is therefore + * not bounded by the amount of positions, and the queue has to be able to grow. + *

+ * + * @param tail the amount of entries the queue currently holds + */ + private void ensureRoom(int tail) { + if (tail == this.queue.length) { + this.queue = java.util.Arrays.copyOf(this.queue, this.queue.length * 2); + } + } + + /** + * Calculates the light of a section from the blocks it holds. + * + * @param opacity the light properties of every block of the section + * @return the calculated light of the section + */ + public LightNibbles propagate(SectionOpacity opacity) { + if (!opacity.hasEmission()) { + return LightNibbles.uniform(0); + } + + Arrays.fill(this.levels, (byte) 0); + int tail = seed(opacity); + int head = 0; + + while (head < tail) { + int index = this.queue[head++]; + int level = this.levels[index]; + + if (level <= 1) { + continue; + } + + int x = index & MASK; + int z = (index >> 4) & MASK; + int y = (index >> 8) & MASK; + int next = level - 1; + + for (BlockFace face : FACES) { + int neighbourX = x + face.offsetX(); + int neighbourY = y + face.offsetY(); + int neighbourZ = z + face.offsetZ(); + + if (isOutside(neighbourX, neighbourY, neighbourZ)) { + continue; + } + // Only the face light enters decides whether it can pass. Testing the face it + // leaves as well would keep every emitting block that is opaque itself dark, and a + // glowstone block is exactly that. + if (opacity.blocksFace(neighbourX, neighbourY, neighbourZ, face.opposite())) { + continue; + } + + int neighbourIndex = index(neighbourX, neighbourY, neighbourZ); + + if (this.levels[neighbourIndex] >= next) { + continue; + } + this.levels[neighbourIndex] = (byte) next; + ensureRoom(tail); + this.queue[tail++] = neighbourIndex; + } + } + return collect(); + } + + /** + * Puts every emitting block of the section into the queue. + * + * @param opacity the light properties of every block of the section + * @return the amount of queued positions + */ + private int seed(SectionOpacity opacity) { + int tail = 0; + + for (int y = 0; y < LightNibbles.DIMENSION; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + int emission = opacity.emission(x, y, z); + + if (emission <= 0) { + continue; + } + int index = index(x, y, z); + this.levels[index] = (byte) emission; + ensureRoom(tail); + this.queue[tail++] = index; + } + } + } + return tail; + } + + /** + * Transfers the calculated levels into a light section. + * A result in which every block carries the same level is stored without an array. + * + * @return the calculated light of the section + */ + private LightNibbles collect() { + return LightNibbles.ofLevels(this.levels, 0); + } + + /** + * Checks whether the given position lies outside of the section. + * + * @param x the x coordinate to check + * @param y the y coordinate to check + * @param z the z coordinate to check + * @return true if the position is outside of the section, otherwise false + */ + private static boolean isOutside(int x, int y, int z) { + return (x | y | z) < 0 || x >= LightNibbles.DIMENSION || y >= LightNibbles.DIMENSION || z >= LightNibbles.DIMENSION; + } + + /** + * Calculates the index of a block inside the section. + * + * @param x the x coordinate inside the section + * @param y the y coordinate inside the section + * @param z the z coordinate inside the section + * @return the index of the block + */ + private static int index(int x, int y, int z) { + return (y << 8) | (z << 4) | x; + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/light/MinestomBlockLightSource.java b/src/main/java/net/theevilreaper/aves/instance/light/MinestomBlockLightSource.java new file mode 100644 index 00000000..39559c7f --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/light/MinestomBlockLightSource.java @@ -0,0 +1,92 @@ +package net.theevilreaper.aves.instance.light; + +import net.minestom.server.instance.block.Block; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.Nullable; + +/** + * The {@link MinestomBlockLightSource} class answers the light properties of a block from the + * registry of the running server. + *

+ * It is the only part of the light engine which knows about Minestom. The propagation itself works + * against {@link BlockLightSource}, so the algorithm can be verified without a server and this + * class carries the whole dependency on the block registry. + *

+ *

+ * A state id the registry does not know is treated as fully transparent and without emission. The + * alternative would be to fail during a propagation, which would cost the light of a whole section + * over a single unknown block. + *

+ *

+ * This type is experimental. The light engine is new and its API may still change. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class MinestomBlockLightSource implements BlockLightSource { + + private static final net.minestom.server.instance.block.BlockFace[] SERVER_FACES = + net.minestom.server.instance.block.BlockFace.values(); + + /** + * {@inheritDoc} + */ + @Override + public int emission(int stateId) { + Block block = resolve(stateId); + return block == null ? 0 : block.registry().lightEmission(); + } + + /** + * {@inheritDoc} + */ + @Override + public boolean blocksFace(int stateId, BlockFace face) { + Block block = resolve(stateId); + + if (block == null) { + return false; + } + return block.registry().occlusionShape().isFaceFull(toServerFace(face)); + } + + /** + * Maps a face of this engine onto the matching face of the server. + * + * @param face the face to map + * @return the matching face of the server + */ + @Contract(pure = true) + private static net.minestom.server.instance.block.BlockFace toServerFace(BlockFace face) { + return SERVER_FACES[face.ordinal()]; + } + + /** + * Resolves the block which belongs to the given state id. + *

+ * The lookup of the server indexes an array without checking its bounds, so a state id outside + * of the known range throws instead of reporting an unknown block. The failure is turned into + * an absent block here, because a propagation must not lose a whole section over one unknown + * state. + *

+ * + * @param stateId the state id to resolve + * @return the block or null if the registry does not know the state + */ + @Contract(pure = true) + private static @Nullable Block resolve(int stateId) { + if (stateId < 0) { + return null; + } + + try { + return Block.fromStateId(stateId); + } catch (IndexOutOfBoundsException exception) { + return null; + } + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/light/SectionOpacity.java b/src/main/java/net/theevilreaper/aves/instance/light/SectionOpacity.java new file mode 100644 index 00000000..a1d7d2c5 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/light/SectionOpacity.java @@ -0,0 +1,394 @@ +package net.theevilreaper.aves.instance.light; + +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; +import org.jetbrains.annotations.Nullable; + +import java.util.Arrays; + +/** + * The {@link SectionOpacity} class holds the light properties of every block of a section in a form + * the propagation can read without touching a registry. + *

+ * Resolving the properties of a block is the dominant cost of a light propagation, because a + * breadth-first search visits the same block from up to six directions and would otherwise resolve + * it again every time. This class resolves every distinct block state once when the table is built + * and answers from two arrays afterwards. + *

+ *

+ * The occlusion of a block is stored per face. A block which occludes only some of its faces, such + * as a slab or a stair, is common enough that a single flag per block would produce visibly wrong + * light. + *

+ *

+ * This type is experimental. The light engine is new and its API may still change. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +public final class SectionOpacity { + + private static final BlockFace[] FACES = BlockFace.values(); + + /** + * The marker a uniformity scan reports when a section holds more than one state. + */ + private static final int NOT_UNIFORM = Integer.MIN_VALUE; + + /** + * The amount of bits the occluded faces of a block occupy in a resolved state. + * One bit per face, so a resolved state fits into a short together with its emission. + */ + private static final int OCCLUSION_BITS = 6; + + /** + * The bits which carry the occluded faces of a resolved state. + */ + private static final int OCCLUSION_MASK = (1 << OCCLUSION_BITS) - 1; + + private final byte @Nullable [] occlusion; + private final byte @Nullable [] emission; + private final byte uniformOcclusion; + private final byte uniformEmission; + private final boolean hasEmission; + private final boolean fullyTransparent; + + /** + * Creates a new table from the given values. + * + * @param occlusion the occluded faces of every block, or null for a uniform section + * @param emission the emitted light level of every block, or null for a uniform section + * @param uniformOcclusion the occluded faces of a uniform section + * @param uniformEmission the emitted light level of a uniform section + * @param hasEmission whether any block of the section emits light + * @param fullyTransparent whether no block of the section occludes any face + */ + private SectionOpacity(byte @Nullable [] occlusion, byte @Nullable [] emission, + byte uniformOcclusion, byte uniformEmission, + boolean hasEmission, boolean fullyTransparent) { + this.occlusion = occlusion; + this.emission = emission; + this.uniformOcclusion = uniformOcclusion; + this.uniformEmission = uniformEmission; + this.hasEmission = hasEmission; + this.fullyTransparent = fullyTransparent; + } + + /** + * Builds the table for a section from the state ids of its blocks. + * Every distinct state id is resolved exactly once. + * + * @param stateIds the state id of every block of the section + * @param source the source which describes the light properties of a block + * @return the created table + * @throws IllegalArgumentException if the given array does not cover the whole section + */ + public static SectionOpacity of(int[] stateIds, BlockLightSource source) { + if (stateIds.length != LightNibbles.BLOCK_COUNT) { + throw new IllegalArgumentException( + "A section holds " + LightNibbles.BLOCK_COUNT + " blocks but the given array holds " + stateIds.length + ); + } + + // A section of one repeated state needs no table at all. Whole sections of a world are + // exactly that, so the shortcut saves both the per position lookups and the two arrays. The + // scan stops at the first differing block, which makes it free for every other section. + int uniform = uniformStateOf(stateIds); + + if (uniform != NOT_UNIFORM) { + int properties = resolve(uniform, source); + byte occluded = (byte) (properties & OCCLUSION_MASK); + byte emitted = (byte) (properties >>> OCCLUSION_BITS); + return new SectionOpacity(null, null, occluded, emitted, emitted != 0, occluded == 0); + } + + byte[] occlusion = new byte[stateIds.length]; + byte[] emission = new byte[stateIds.length]; + StateCache cache = new StateCache(); + int anyEmission = 0; + int anyOcclusion = 0; + + // Blocks of a world come in runs, so the state of a block is very often the state of the + // one before it. Remembering the last one turns the lookup of a run into a comparison. + int previousState = NOT_UNIFORM; + int previousProperties = 0; + + for (int index = 0; index < stateIds.length; index++) { + int stateId = stateIds[index]; + int properties = previousProperties; + + if (stateId != previousState) { + properties = cache.propertiesOf(stateId, source); + previousState = stateId; + previousProperties = properties; + } + int occluded = properties & OCCLUSION_MASK; + int emitted = properties >>> OCCLUSION_BITS; + occlusion[index] = (byte) occluded; + emission[index] = (byte) emitted; + anyEmission |= emitted; + anyOcclusion |= occluded; + } + return new SectionOpacity(occlusion, emission, (byte) 0, (byte) 0, anyEmission != 0, anyOcclusion == 0); + } + + /** + * Resolves the light properties of a single block state. + *

+ * The occluded faces and the emission are returned as one value rather than as a pair, because + * a pair would have to be an object and this method is called once per distinct state of every + * section of every chunk a server lights. + *

+ * + * @param stateId the state id to resolve + * @param source the source which describes the light properties of a block + * @return the occluded faces in the low bits and the emission above them + */ + @Contract(pure = true) + private static int resolve(int stateId, BlockLightSource source) { + int mask = 0; + + for (BlockFace face : FACES) { + if (source.blocksFace(stateId, face)) { + mask |= 1 << face.ordinal(); + } + } + return mask | (source.emission(stateId) << OCCLUSION_BITS); + } + + /** + * The {@link StateCache} class remembers the resolved properties of every state a table build + * has already seen. + *

+ * A section holds 4096 blocks but only a handful of distinct states, and resolving one of them + * reaches into the block registry of the server. The build therefore asks this cache once per + * block and the registry once per distinct state. + *

+ *

+ * The cache is a table with linear probing rather than a {@link java.util.HashMap}, for one + * reason: a map is keyed by objects. Its key would be a boxed state id and its value would be a + * pair object, and both would be created per block rather than per distinct state. That cost + * was measured and it dominated the build. Two flat arrays and a packed short have no such + * cost, and the whole cache dies with the build that created it. + *

+ */ + private static final class StateCache { + + /** + * The amount of slots a cache starts with. Enough for a section of a real world, which + * rarely holds more than a few dozen distinct states. + */ + private static final int INITIAL_SLOTS = 64; + + /** + * The marker an unused slot carries. + *

+ * A state id of exactly this value would be mistaken for an empty slot. The lowest possible + * integer is not a state id any registry produces, and the uniformity scan of the table + * already reserves it for the same reason. + *

+ */ + private static final int EMPTY = Integer.MIN_VALUE; + + private int[] keys; + private short[] properties; + private int size; + + /** + * Creates an empty cache. + */ + private StateCache() { + this.keys = new int[INITIAL_SLOTS]; + this.properties = new short[INITIAL_SLOTS]; + Arrays.fill(this.keys, EMPTY); + } + + /** + * Returns the resolved properties of the given state, resolving it if this is the first + * time the state is seen. + * + * @param stateId the state id to look up + * @param source the source which describes the light properties of a block + * @return the occluded faces in the low bits and the emission above them + */ + private int propertiesOf(int stateId, BlockLightSource source) { + int[] table = this.keys; + int mask = table.length - 1; + int slot = spread(stateId) & mask; + int key = table[slot]; + + while (key != stateId) { + if (key == EMPTY) { + return insert(slot, stateId, source); + } + slot = (slot + 1) & mask; + key = table[slot]; + } + return this.properties[slot]; + } + + /** + * Resolves a state which was seen for the first time and stores it in the given free slot. + * + * @param slot the free slot the probe ended on + * @param stateId the state id to resolve + * @param source the source which describes the light properties of a block + * @return the occluded faces in the low bits and the emission above them + */ + private int insert(int slot, int stateId, BlockLightSource source) { + short resolved = (short) resolve(stateId, source); + this.keys[slot] = stateId; + this.properties[slot] = resolved; + this.size++; + + // Linear probing degrades once a table fills up, so it is grown well before it is full. + if (this.size * 2 >= this.keys.length) { + grow(); + } + return resolved; + } + + /** + * Doubles the amount of slots and moves every stored state into the larger table. + */ + private void grow() { + int[] oldKeys = this.keys; + short[] oldProperties = this.properties; + int[] newKeys = new int[oldKeys.length * 2]; + short[] newProperties = new short[oldKeys.length * 2]; + int mask = newKeys.length - 1; + Arrays.fill(newKeys, EMPTY); + + for (int index = 0; index < oldKeys.length; index++) { + int key = oldKeys[index]; + + if (key == EMPTY) { + continue; + } + int slot = spread(key) & mask; + + while (newKeys[slot] != EMPTY) { + slot = (slot + 1) & mask; + } + newKeys[slot] = key; + newProperties[slot] = oldProperties[index]; + } + this.keys = newKeys; + this.properties = newProperties; + } + + /** + * Mixes the bits of a state id so that neighbouring ids do not end up in neighbouring slots. + *

+ * A table indexed by a power of two only ever looks at the low bits of its key. State ids of + * one block are consecutive, so without mixing a section of a single block type would fill + * one run of slots and probe through all of it. + *

+ * + * @param stateId the state id to mix + * @return the mixed value + */ + @Contract(pure = true) + private static int spread(int stateId) { + int mixed = stateId * 0x9E3779B9; + return mixed ^ (mixed >>> 16); + } + } + + /** + * Checks whether light is unable to pass the given face of the block at the given position. + * + * @param x the x coordinate inside the section + * @param y the y coordinate inside the section + * @param z the z coordinate inside the section + * @param face the face to check + * @return true if light cannot pass the face, otherwise false + */ + @Contract(pure = true) + public boolean blocksFace(int x, int y, int z, BlockFace face) { + byte[] table = this.occlusion; + byte mask = table == null ? this.uniformOcclusion : table[index(x, y, z)]; + return (mask & (1 << face.ordinal())) != 0; + } + + /** + * Returns the amount of light the block at the given position emits. + * + * @param x the x coordinate inside the section + * @param y the y coordinate inside the section + * @param z the z coordinate inside the section + * @return the emitted light level of the block + */ + @Contract(pure = true) + public int emission(int x, int y, int z) { + byte[] table = this.emission; + return table == null ? this.uniformEmission : table[index(x, y, z)]; + } + + /** + * Checks whether any block of the section emits light. + * A section without an emitting block needs no block light propagation at all. + * + * @return true if a block of the section emits light, otherwise false + */ + @Contract(pure = true) + public boolean hasEmission() { + return this.hasEmission; + } + + /** + * Checks whether no block of the section occludes any face. + * Light travels through such a section without any obstacle. + * + * @return true if the section occludes nothing, otherwise false + */ + @Contract(pure = true) + public boolean isFullyTransparent() { + return this.fullyTransparent; + } + + /** + * Checks whether every block of the section holds the same state. + * Such a section carries no per position table. + * + * @return true if the section holds a single state, otherwise false + */ + @Contract(pure = true) + public boolean isUniform() { + return this.occlusion == null; + } + + /** + * Determines whether every block of the given section holds the same state. + * + * @param stateIds the state id of every block of the section + * @return the repeated state id, or {@link #NOT_UNIFORM} if the section holds more than one + */ + @Contract(pure = true) + private static int uniformStateOf(int[] stateIds) { + int first = stateIds[0]; + + for (int stateId : stateIds) { + if (stateId != first) { + return NOT_UNIFORM; + } + } + return first; + } + + /** + * Calculates the index of a block inside the section. + * + * @param x the x coordinate inside the section + * @param y the y coordinate inside the section + * @param z the z coordinate inside the section + * @return the index of the block + */ + @Contract(pure = true) + private static int index(int x, int y, int z) { + return (y << 8) | (z << 4) | x; + } +} diff --git a/src/main/java/net/theevilreaper/aves/instance/light/package-info.java b/src/main/java/net/theevilreaper/aves/instance/light/package-info.java new file mode 100644 index 00000000..af50ee8b --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/instance/light/package-info.java @@ -0,0 +1,4 @@ +@NotNullByDefault +package net.theevilreaper.aves.instance.light; + +import org.jetbrains.annotations.NotNullByDefault; diff --git a/src/main/java/net/theevilreaper/aves/map/provider/AbstractMapProvider.java b/src/main/java/net/theevilreaper/aves/map/provider/AbstractMapProvider.java index 0f653ca8..d39e475c 100644 --- a/src/main/java/net/theevilreaper/aves/map/provider/AbstractMapProvider.java +++ b/src/main/java/net/theevilreaper/aves/map/provider/AbstractMapProvider.java @@ -12,6 +12,7 @@ import net.minestom.server.instance.Instance; import net.minestom.server.instance.InstanceContainer; import net.minestom.server.instance.anvil.AnvilLoader; +import org.jetbrains.annotations.ApiStatus; import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.UnmodifiableView; import org.slf4j.Logger; @@ -35,13 +36,20 @@ *

* * @author theEvilReaper - * @version 1.2.0 + * @version 1.3.0 * @since 1.6.0 */ public abstract class AbstractMapProvider implements MapProvider { private static final Logger MAP_LOGGER = LoggerFactory.getLogger(AbstractMapProvider.class); + /** + * The factory which is used when a provider does not choose a chunk loader itself. + * It keeps the loader which Minestom ships with so existing providers behave as before. + */ + private static final ChunkLoaderFactory DEFAULT_CHUNK_LOADER_FACTORY = + (mapEntry, dimension) -> new AnvilLoader(mapEntry.getDirectoryRoot(), dimension); + private final PathFilter mapFilter; protected final FileHandler fileHandler; protected final List mapEntries; @@ -81,7 +89,30 @@ protected void registerInstance(InstanceContainer instance, MapEntry mapEntry) { * @param dimensionKey the dimension type key for the instance */ protected void registerInstance(InstanceContainer instance, MapEntry mapEntry, RegistryKey dimensionKey) { - instance.setChunkLoader(new AnvilLoader(mapEntry.getDirectoryRoot(), dimensionKey.key())); + this.registerInstance(instance, mapEntry, dimensionKey, DEFAULT_CHUNK_LOADER_FACTORY); + } + + /** + * Registers the specified map entry as an active instance in the server. + * Sets up chunk loading and time rate, and registers the instance with the server manager. + *

+ * The chunk loader is created by the given factory which allows a provider to replace the + * default loader, for example with {@link ChunkLoaderFactory#anvil()}. + *

+ * + * @param instance to be registered + * @param mapEntry representing the folder that contains the map files + * @param dimensionKey the dimension type key for the instance + * @param loaderFactory the factory which creates the chunk loader of the instance + */ + @ApiStatus.Experimental + protected void registerInstance( + InstanceContainer instance, + MapEntry mapEntry, + RegistryKey dimensionKey, + ChunkLoaderFactory loaderFactory + ) { + instance.setChunkLoader(loaderFactory.create(mapEntry, dimensionKey.key())); instance.enableAutoChunkLoad(true); var defaultClock = instance.defaultClock(); if (defaultClock != null) { diff --git a/src/main/java/net/theevilreaper/aves/map/provider/ChunkLoaderFactory.java b/src/main/java/net/theevilreaper/aves/map/provider/ChunkLoaderFactory.java new file mode 100644 index 00000000..d4310ef5 --- /dev/null +++ b/src/main/java/net/theevilreaper/aves/map/provider/ChunkLoaderFactory.java @@ -0,0 +1,49 @@ +package net.theevilreaper.aves.map.provider; + +import net.kyori.adventure.key.Key; +import net.minestom.server.instance.ChunkLoader; +import net.theevilreaper.aves.instance.anvil.AvesAnvilLoader; +import net.theevilreaper.aves.map.MapEntry; +import org.jetbrains.annotations.ApiStatus; +import org.jetbrains.annotations.Contract; + +/** + * The {@link ChunkLoaderFactory} interface creates the {@link ChunkLoader} which a map provider + * assigns to an instance. + *

+ * The factory exists so a provider can choose its loader instead of being tied to a single + * implementation. Existing code keeps the loader of Minestom while new code can opt into the + * loader of Aves without any change to the provider itself. + *

+ *

+ * This type is experimental. It is introduced together with the Anvil loader of Aves and its + * API may still change while that loader is being validated against real worlds. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ApiStatus.Experimental +@FunctionalInterface +public interface ChunkLoaderFactory { + + /** + * Returns a factory which creates the Anvil loader of Aves. + * + * @return the factory for the Anvil loader of Aves + */ + @Contract(pure = true) + static ChunkLoaderFactory anvil() { + return (mapEntry, dimension) -> new AvesAnvilLoader(mapEntry.getDirectoryRoot(), dimension); + } + + /** + * Creates the chunk loader for the given map entry. + * + * @param mapEntry the entry which describes the directory of the map + * @param dimension the key of the dimension the instance uses + * @return the created chunk loader + */ + ChunkLoader create(MapEntry mapEntry, Key dimension); +} diff --git a/src/test/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnosticsConcurrencyTest.java b/src/test/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnosticsConcurrencyTest.java new file mode 100644 index 00000000..ea4cbe8c --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnosticsConcurrencyTest.java @@ -0,0 +1,270 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicIntegerArray; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Stresses the throttling and the counters of the diagnostics from many threads at once. + *

+ * A loader reports from every thread which loads or saves a chunk, so the throttling has to elect + * exactly one reporter per distinct name no matter how many threads compete for it. Losing that + * property floods the log of a broken world with thousands of identical lines, and losing an + * increment makes the summary of a shutdown report fewer chunks than were really processed. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class AnvilDiagnosticsConcurrencyTest { + + /** + * The time a latch is waited for before the test is considered stuck. + */ + private static final long AWAIT_SECONDS = 60L; + + @Test + void testConcurrentReportsElectExactlyOneWinnerForEveryName() throws InterruptedException, ExecutionException { + // Every thread reports every name, only in a different order. A tracking set which loses an + // update lets two threads believe they were the first for the same name, which is what + // turns a single warning into one warning per chunk of a broken world. + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + int threadCount = 16; + int nameCount = 32; + AtomicIntegerArray blockWinners = new AtomicIntegerArray(nameCount); + AtomicIntegerArray biomeWinners = new AtomicIntegerArray(nameCount); + CountDownLatch start = new CountDownLatch(1); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(threadCount); + + for (int thread = 0; thread < threadCount; thread++) { + int offset = thread; + futures.add(executor.submit(() -> { + awaitStart(start); + + for (int step = 0; step < nameCount; step++) { + int name = (offset + step) % nameCount; + + if (diagnostics.reportUnknownBlock("aves:block_" + name)) { + blockWinners.incrementAndGet(name); + } + if (diagnostics.reportUnknownBiome("aves:biome_" + name)) { + biomeWinners.incrementAndGet(name); + } + } + return null; + })); + } + start.countDown(); + awaitAll(futures); + } + + for (int name = 0; name < nameCount; name++) { + assertEquals(1, blockWinners.get(name), "the block name " + name + " was reported by more than one thread"); + assertEquals(1, biomeWinners.get(name), "the biome name " + name + " was reported by more than one thread"); + } + assertEquals(nameCount, diagnostics.unknownBlockCount()); + assertEquals(nameCount, diagnostics.unknownBiomeCount()); + } + + @Test + void testConcurrentReportsAreRejectedOnceTheCapIsReached() throws InterruptedException, ExecutionException { + // The cap is filled before the threads start, so the state every thread observes is defined + // by the happens before edge the executor establishes. Not a single further name may be + // tracked afterwards, which is the property that keeps the heap of a server bounded when a + // broken world holds an unlimited amount of unknown names. + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + int threadCount = 32; + + for (int name = 0; name < AnvilDiagnostics.MAX_TRACKED_NAMES; name++) { + assertTrue(diagnostics.reportUnknownBlock("aves:filler_" + name)); + } + + CountDownLatch start = new CountDownLatch(1); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(threadCount); + + for (int thread = 0; thread < threadCount; thread++) { + int name = thread; + futures.add(executor.submit(() -> { + awaitStart(start); + return diagnostics.reportUnknownBlock("aves:late_" + name); + })); + } + start.countDown(); + + int winners = 0; + + for (Future future : futures) { + if (future.get()) { + winners++; + } + } + assertEquals(0, winners, "no report may pass once the cap is reached"); + } + assertEquals(AnvilDiagnostics.MAX_TRACKED_NAMES, diagnostics.unknownBlockCount()); + } + + @Test + void testConcurrentReportsKeepTheTrackedNamesBounded() throws InterruptedException, ExecutionException { + // The threads race for the cap from an empty state with far more names than the cap allows. + // The cap is a check followed by an insert and is therefore a soft one under concurrency: + // every thread which is between the check and the insert can add one name beyond it, so the + // set may hold up to one extra name per thread. What must never happen is that the set + // grows towards the amount of offered names, and that two threads win the same name. + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + int threadCount = 32; + int namesPerThread = 64; + CountDownLatch start = new CountDownLatch(1); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(threadCount); + + for (int thread = 0; thread < threadCount; thread++) { + int owner = thread; + futures.add(executor.submit(() -> { + awaitStart(start); + int accepted = 0; + + for (int name = 0; name < namesPerThread; name++) { + // Every thread owns its own names, so a name can only be won once and the + // amount of winners has to match the size of the tracking set exactly. + if (diagnostics.reportUnknownBlock("aves:" + owner + "_" + name)) { + accepted++; + } + } + return accepted; + })); + } + start.countDown(); + + int winners = 0; + + for (Future future : futures) { + winners += future.get(); + } + assertEquals(diagnostics.unknownBlockCount(), winners, "every tracked name has to have exactly one winner"); + } + + int tracked = diagnostics.unknownBlockCount(); + + assertTrue(tracked >= AnvilDiagnostics.MAX_TRACKED_NAMES, "the cap has to be filled but only " + tracked + " names were tracked"); + assertTrue( + tracked <= AnvilDiagnostics.MAX_TRACKED_NAMES + threadCount, + "the cap may be exceeded by at most one name per racing thread but " + tracked + " names were tracked" + ); + } + + @Test + void testConcurrentOnceOnlyFlagsElectExactlyOneWinner() throws InterruptedException, ExecutionException { + // Both flags exist so a world which was generated elsewhere logs its problem once instead of + // once per chunk. Two winners mean the flag is not a compare and set anymore. + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + int threadCount = 64; + AtomicIntegerArray winners = new AtomicIntegerArray(2); + CountDownLatch start = new CountDownLatch(1); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(threadCount); + + for (int thread = 0; thread < threadCount; thread++) { + futures.add(executor.submit(() -> { + awaitStart(start); + + if (diagnostics.reportPartialChunk()) { + winners.incrementAndGet(0); + } + if (diagnostics.reportSectionOutOfRange()) { + winners.incrementAndGet(1); + } + return null; + })); + } + start.countDown(); + awaitAll(futures); + } + + assertEquals(1, winners.get(0), "a partial chunk may be reported by exactly one thread"); + assertEquals(1, winners.get(1), "a section outside of the world may be reported by exactly one thread"); + } + + @Test + void testConcurrentCountingKeepsEveryCounterExact() throws InterruptedException, ExecutionException { + // The summary of a shutdown must not lose a chunk. Every thread touches all three counters + // in the same run, which is what a loader does while it loads, saves and fails at once. + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + int threadCount = 16; + int perThread = 2000; + CountDownLatch start = new CountDownLatch(1); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(threadCount); + + for (int thread = 0; thread < threadCount; thread++) { + futures.add(executor.submit(() -> { + awaitStart(start); + + for (int step = 0; step < perThread; step++) { + diagnostics.countChunkLoaded(); + + if (step % 2 == 0) { + diagnostics.countChunkSaved(); + } + if (step % 4 == 0) { + diagnostics.countError(); + } + } + return null; + })); + } + start.countDown(); + awaitAll(futures); + } + + assertEquals((long) threadCount * perThread, diagnostics.chunksLoaded()); + assertEquals((long) threadCount * perThread / 2, diagnostics.chunksSaved()); + assertEquals((long) threadCount * perThread / 4, diagnostics.errors()); + } + + /** + * Waits for the given latch and fails when it is not released in time. + * + * @param latch the latch to wait for + */ + private static void awaitStart(CountDownLatch latch) { + try { + assertTrue(latch.await(AWAIT_SECONDS, TimeUnit.SECONDS), "a worker waited too long for its barrier"); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + fail("a worker was interrupted while it waited for its barrier"); + } + } + + /** + * Waits for every given task and propagates the failure of the first broken one. + * + * @param futures the tasks to wait for + * @throws InterruptedException if the waiting thread is interrupted + * @throws ExecutionException if a task failed + */ + private static void awaitAll(List> futures) throws InterruptedException, ExecutionException { + for (Future future : futures) { + future.get(); + } + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnosticsTest.java b/src/test/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnosticsTest.java new file mode 100644 index 00000000..62f01d49 --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/anvil/AnvilDiagnosticsTest.java @@ -0,0 +1,151 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the diagnostics which throttle repeating warnings and collect the counters for the + * summary of a loader. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class AnvilDiagnosticsTest { + + @Test + void testTheFirstReportOfANameIsAllowed() { + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + + assertTrue(diagnostics.reportUnknownBlock("minecraft:custom")); + } + + @Test + void testARepeatedNameIsSuppressed() { + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + diagnostics.reportUnknownBlock("minecraft:custom"); + + assertFalse(diagnostics.reportUnknownBlock("minecraft:custom")); + } + + @Test + void testDifferentNamesAreReportedSeparately() { + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + + assertTrue(diagnostics.reportUnknownBlock("minecraft:a")); + assertTrue(diagnostics.reportUnknownBlock("minecraft:b")); + assertEquals(2, diagnostics.unknownBlockCount()); + } + + @Test + void testBlocksAndBiomesUseSeparateBudgets() { + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + diagnostics.reportUnknownBlock("minecraft:shared"); + + assertTrue(diagnostics.reportUnknownBiome("minecraft:shared")); + } + + @Test + void testTheTrackedNamesAreCappedToProtectTheHeap() { + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + + for (int i = 0; i < AnvilDiagnostics.MAX_TRACKED_NAMES; i++) { + assertTrue(diagnostics.reportUnknownBlock("minecraft:block_" + i)); + } + + assertFalse(diagnostics.reportUnknownBlock("minecraft:one_too_many")); + assertEquals(AnvilDiagnostics.MAX_TRACKED_NAMES, diagnostics.unknownBlockCount()); + } + + @Test + void testAOnceOnlyFlagOnlyPassesTheFirstTime() { + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + + assertTrue(diagnostics.reportPartialChunk()); + assertFalse(diagnostics.reportPartialChunk()); + } + + @Test + void testTheCountersStartAtZero() { + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + + assertEquals(0, diagnostics.chunksLoaded()); + assertEquals(0, diagnostics.chunksSaved()); + assertEquals(0, diagnostics.errors()); + } + + @Test + void testTheCountersAddUp() { + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + diagnostics.countChunkLoaded(); + diagnostics.countChunkLoaded(); + diagnostics.countChunkSaved(); + diagnostics.countError(); + + assertEquals(2, diagnostics.chunksLoaded()); + assertEquals(1, diagnostics.chunksSaved()); + assertEquals(1, diagnostics.errors()); + } + + @Test + void testConcurrentReportsElectExactlyOneWinnerPerName() throws InterruptedException, ExecutionException { + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + int threadCount = 32; + CountDownLatch start = new CountDownLatch(1); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(threadCount); + + for (int i = 0; i < threadCount; i++) { + futures.add(executor.submit(() -> { + start.await(); + return diagnostics.reportUnknownBlock("minecraft:contested"); + })); + } + start.countDown(); + + int winners = 0; + + for (Future future : futures) { + if (future.get()) { + winners++; + } + } + assertEquals(1, winners, "exactly one thread may report the same name"); + } + } + + @Test + void testConcurrentCountingLosesNoIncrement() throws InterruptedException, ExecutionException { + AnvilDiagnostics diagnostics = new AnvilDiagnostics(); + int threadCount = 16; + int perThread = 500; + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(threadCount); + + for (int i = 0; i < threadCount; i++) { + futures.add(executor.submit(() -> { + for (int j = 0; j < perThread; j++) { + diagnostics.countChunkLoaded(); + } + })); + } + for (Future future : futures) { + future.get(); + } + } + assertEquals((long) threadCount * perThread, diagnostics.chunksLoaded()); + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoaderConcurrencyTest.java b/src/test/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoaderConcurrencyTest.java new file mode 100644 index 00000000..ef75c6d1 --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoaderConcurrencyTest.java @@ -0,0 +1,473 @@ +package net.theevilreaper.aves.instance.anvil; + +import net.kyori.adventure.key.Key; +import net.minestom.server.MinecraftServer; +import net.minestom.server.exception.ExceptionHandler; +import net.minestom.server.instance.Chunk; +import net.minestom.server.instance.Instance; +import net.minestom.server.instance.block.Block; +import net.minestom.testing.Env; +import net.minestom.testing.extension.MicrotusExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Stresses the chunk loader with several region files which are used from many threads at once. + *

+ * The loader reports parallel loading and parallel saving as supported, so a running server hands it + * work from many threads. The tests here verify the two properties that promise depends on: no chunk + * may be lost while other chunks of the same region file are written, and the amount of region files + * the loader keeps open has to stay below its limit even when every thread opens a new one. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ExtendWith(MicrotusExtension.class) +class AvesAnvilLoaderConcurrencyTest { + + private static final Key OVERWORLD = Key.key("minecraft:overworld"); + + /** + * The time a latch is waited for before the test is considered stuck. + */ + private static final long AWAIT_SECONDS = 60L; + + /** + * The amount of region files the chunks of the first test are spread over. + */ + private static final int REGION_COUNT = 4; + + /** + * The amount of chunks every region file of the first test holds. + */ + private static final int CHUNKS_PER_REGION = 4; + + @TempDir + private Path worldRoot; + + /** + * Creates a loader for the temporary world of the test. + * + * @return the created loader + */ + private AvesAnvilLoader loader() { + return new AvesAnvilLoader(this.worldRoot, OVERWORLD); + } + + @Test + void testConcurrentSavesAndLoadsOverSeveralRegionsLoseNoChunk(Env env) throws IOException, InterruptedException, ExecutionException { + // Half of the chunks are already on disk and are only read while the other half is written. + // Reading and writing therefore meet inside the same region files, which is the situation a + // server produces while it streams chunks in and out. A region file which mixed the two + // would either hand back a payload of the wrong chunk or fail to decompress it, and both + // show up as a missing marker block or as a counted error. + Instance instance = env.createEmptyInstance(loader()); + List chunks = createChunks(instance); + List stored = new ArrayList<>(); + List> groups = new ArrayList<>(); + + for (int region = 0; region < REGION_COUNT; region++) { + List group = new ArrayList<>(); + + for (int local = 0; local < CHUNKS_PER_REGION; local++) { + Chunk chunk = chunks.get(region * CHUNKS_PER_REGION + local); + + if (local < CHUNKS_PER_REGION / 2) { + stored.add(chunk); + } else { + group.add(chunk); + } + } + groups.add(group); + } + + CountDownLatch start = new CountDownLatch(1); + + try (AvesAnvilLoader loader = loader(); + ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + for (Chunk chunk : stored) { + loader.saveChunk(chunk); + } + + List> futures = new ArrayList<>(groups.size() + stored.size()); + + for (List group : groups) { + futures.add(executor.submit(() -> { + awaitStart(start); + loader.saveChunks(group); + return null; + })); + } + for (Chunk chunk : stored) { + futures.add(executor.submit(() -> { + awaitStart(start); + assertNotNull( + loader.loadChunk(instance, chunk.getChunkX(), chunk.getChunkZ()), + "the chunk " + chunk.getChunkX() + "/" + chunk.getChunkZ() + " vanished while other chunks were written" + ); + return null; + })); + } + start.countDown(); + awaitAll(futures); + + assertEquals(0, loader.diagnostics().errors(), "no chunk may fail while the loader works in parallel"); + assertEquals(chunks.size(), loader.diagnostics().chunksSaved()); + assertEquals(stored.size(), loader.diagnostics().chunksLoaded()); + assertEquals(REGION_COUNT, loader.openRegionCount(), "the loader has to hold exactly one file per region"); + assertTrue(loader.openRegionCount() <= AvesAnvilLoader.DEFAULT_OPEN_REGION_LIMIT); + } + + try (AvesAnvilLoader reader = loader()) { + for (int index = 0; index < chunks.size(); index++) { + Chunk chunk = chunks.get(index); + Chunk loaded = reader.loadChunk(instance, chunk.getChunkX(), chunk.getChunkZ()); + + assertNotNull(loaded, "the chunk " + chunk.getChunkX() + "/" + chunk.getChunkZ() + " is missing"); + assertMarker(loaded, index); + } + assertEquals(0, reader.diagnostics().errors()); + } + } + + @Test + void testTheOpenRegionLimitHoldsWhileManyThreadsOpenRegions(Env env) throws IOException, InterruptedException, ExecutionException { + // Every thread works in a region file of its own, so every one of them opens a new file and + // forces the loader to evict another one. A limit which is only respected by a single thread + // would let the amount of open files grow with the amount of threads, which is exactly the + // file descriptor leak the limit exists to prevent. + // An eviction drops a file from the cache but never closes it under the thread which is + // writing to it, so every save has to succeed even though the limit is hit constantly. A + // failure would be reported to the exception manager, which the environment turns into a + // failed test on its own. + int regionCount = 8; + int limit = 2; + Instance instance = env.createEmptyInstance(loader()); + List chunks = new ArrayList<>(regionCount); + + for (int region = 0; region < regionCount; region++) { + Chunk chunk = instance.loadChunk(region * 32, 0).join(); + place(chunk, 0, 40, 0, Block.STONE); + chunks.add(chunk); + } + + CountDownLatch start = new CountDownLatch(1); + + try (AvesAnvilLoader loader = new AvesAnvilLoader(this.worldRoot, OVERWORLD, limit)) { + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(regionCount); + + for (Chunk chunk : chunks) { + futures.add(executor.submit(() -> { + awaitStart(start); + loader.saveChunk(chunk); + return null; + })); + } + start.countDown(); + awaitAll(futures); + } + + assertTrue( + loader.openRegionCount() <= limit, + "the loader has to stay below its limit of " + limit + " files but held " + loader.openRegionCount() + ); + assertEquals(0, loader.diagnostics().errors(), "an eviction may not make the save of another thread fail"); + assertEquals(regionCount, loader.diagnostics().chunksSaved(), "every chunk has to be written"); + assertTrue(loader.openRegionCount() <= limit, "the limit has to hold after the concurrent pass as well"); + } + + try (AvesAnvilLoader reader = loader()) { + for (Chunk chunk : chunks) { + assertNotNull( + reader.loadChunk(instance, chunk.getChunkX(), chunk.getChunkZ()), + "the chunk " + chunk.getChunkX() + "/" + chunk.getChunkZ() + " was lost" + ); + } + } + } + + @Test + void testLoadingSurvivesTheEvictionOfItsRegionFile(Env env) throws IOException, InterruptedException, ExecutionException { + // Every thread reads from a region file of its own while the limit allows a single open + // file, so every load evicts the file another thread is about to read from. The loader owns + // that eviction, the file on disk is intact, and a read which fails here loses a chunk the + // server would then regenerate over the stored data. + int regionCount = 8; + int rounds = 60; + Instance instance = env.createEmptyInstance(loader()); + List chunks = storeChunkPerRegion(instance, regionCount); + List failures = new CopyOnWriteArrayList<>(); + CountDownLatch start = new CountDownLatch(1); + ExceptionHandler previous = MinecraftServer.getExceptionManager().getExceptionHandler(); + + try (AvesAnvilLoader loader = new AvesAnvilLoader(this.worldRoot, OVERWORLD, 1)) { + // A failed load reports to the exception manager, which the environment turns into a + // failed test before the assertion below could describe what went wrong. + MinecraftServer.getExceptionManager().setExceptionHandler(ignored -> { + }); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(chunks.size()); + + for (Chunk chunk : chunks) { + futures.add(executor.submit(() -> { + awaitStart(start); + readRepeatedly(loader, instance, chunk, rounds, failures, false); + return null; + })); + } + start.countDown(); + awaitAll(futures); + } finally { + MinecraftServer.getExceptionManager().setExceptionHandler(previous); + } + } + assertNoFailure(failures, "a load may not fail because another thread evicted its region file"); + } + + @Test + void testLoadingSurvivesTheUnloadOfAnotherChunkOfTheSameRegion(Env env) throws IOException, InterruptedException, ExecutionException { + // Every chunk of this test lives in the same region file. A thread which unloads the last + // chunk the loader tracks closes that file, and the loader only starts tracking a chunk + // after it has been read, so a reader which is still ahead of its own registration loses + // the handle it already holds. + int chunkCount = 8; + int rounds = 60; + Instance instance = env.createEmptyInstance(loader()); + List chunks = new ArrayList<>(chunkCount); + + for (int chunkX = 0; chunkX < chunkCount; chunkX++) { + Chunk chunk = instance.loadChunk(chunkX, 0).join(); + place(chunk, 0, 40, 0, Block.STONE); + chunks.add(chunk); + } + + try (AvesAnvilLoader writer = loader()) { + writer.saveChunks(chunks); + } + + List failures = new CopyOnWriteArrayList<>(); + CountDownLatch start = new CountDownLatch(1); + ExceptionHandler previous = MinecraftServer.getExceptionManager().getExceptionHandler(); + + try (AvesAnvilLoader loader = loader()) { + MinecraftServer.getExceptionManager().setExceptionHandler(ignored -> { + }); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(chunks.size()); + + for (Chunk chunk : chunks) { + futures.add(executor.submit(() -> { + awaitStart(start); + readRepeatedly(loader, instance, chunk, rounds, failures, true); + return null; + })); + } + start.countDown(); + awaitAll(futures); + } finally { + MinecraftServer.getExceptionManager().setExceptionHandler(previous); + } + } + assertNoFailure(failures, "a load may not fail because another thread unloaded a chunk of the same region"); + } + + /** + * Writes a single chunk into every one of the given amount of region files. + * + * @param instance the instance which owns the chunks + * @param regionCount the amount of region files to fill + * @return the written chunks in the order of their region + * @throws IOException if a chunk cannot be written + */ + private List storeChunkPerRegion(Instance instance, int regionCount) throws IOException { + List chunks = new ArrayList<>(regionCount); + + for (int region = 0; region < regionCount; region++) { + Chunk chunk = instance.loadChunk(region * 32, 0).join(); + place(chunk, 0, 40, 0, Block.STONE); + chunks.add(chunk); + } + + try (AvesAnvilLoader writer = loader()) { + for (Chunk chunk : chunks) { + writer.saveChunk(chunk); + } + } + return chunks; + } + + /** + * Reads the given chunk repeatedly and records every failure instead of throwing it. + * The failures are collected so the assertion of the test can describe all of them instead of + * being replaced by the first exception a worker throws. + * + * @param loader the loader to read through + * @param instance the instance which owns the chunk + * @param chunk the chunk to read + * @param rounds the amount of reads to perform + * @param failures the list which receives the failures + * @param unload whether the chunk is unloaded again after every read + */ + private static void readRepeatedly( + AvesAnvilLoader loader, Instance instance, Chunk chunk, int rounds, List failures, boolean unload + ) { + for (int round = 0; round < rounds; round++) { + try { + Chunk loaded = loader.loadChunk(instance, chunk.getChunkX(), chunk.getChunkZ()); + + if (loaded == null) { + failures.add(new IllegalStateException( + "the chunk " + chunk.getChunkX() + "/" + chunk.getChunkZ() + " was reported as absent" + )); + continue; + } + if (unload) { + loader.unloadChunk(loaded); + } + } catch (Throwable failure) { + failures.add(failure); + } + } + } + + /** + * Fails the calling test when at least one worker recorded a failure. + * + * @param failures the recorded failures + * @param message the message which describes the expectation + */ + private static void assertNoFailure(List failures, String message) { + if (failures.isEmpty()) { + return; + } + fail(message + " but " + failures.size() + " of them did, the first one was: " + failures.getFirst(), failures.getFirst()); + } + + /** + * Creates the chunks of the first test and marks every one of them with its own block. + * The chunks are spread over {@link #REGION_COUNT} region files so the threads of the test meet + * inside the same files instead of working on separate ones. + * + * @param instance the instance which owns the chunks + * @return the created chunks in the order of their index + */ + private static List createChunks(Instance instance) { + List chunks = new ArrayList<>(REGION_COUNT * CHUNKS_PER_REGION); + + for (int region = 0; region < REGION_COUNT; region++) { + for (int local = 0; local < CHUNKS_PER_REGION; local++) { + int chunkX = (region % 2) * 32 + local % 2; + int chunkZ = (region / 2) * 32 + local / 2; + Chunk chunk = instance.loadChunk(chunkX, chunkZ).join(); + + place(chunk, 0, 40, 0, Block.STONE); + place(chunk, chunks.size(), 41, 0, Block.DIRT); + chunks.add(chunk); + } + } + return chunks; + } + + /** + * Verifies that the given chunk carries the marker of the given index and nothing else. + * The marker sits at a different position per chunk, so a chunk which received the payload of + * another one is detected instead of only being detected as present. + * + * @param chunk the chunk to inspect + * @param index the index the chunk was created with + */ + private static void assertMarker(Chunk chunk, int index) { + assertEquals(Block.STONE, blockAt(chunk, 0, 40, 0), "the chunk " + index + " lost its shared marker"); + assertEquals(Block.DIRT, blockAt(chunk, index, 41, 0), "the chunk " + index + " lost its own marker"); + assertEquals( + Block.AIR, blockAt(chunk, (index + 1) % (REGION_COUNT * CHUNKS_PER_REGION), 41, 0), + "the chunk " + index + " carries the marker of another chunk" + ); + } + + /** + * Places a block in the given chunk while holding its write lock. + * + * @param chunk the chunk which receives the block + * @param x the x coordinate inside the chunk + * @param y the y coordinate of the block + * @param z the z coordinate inside the chunk + * @param block the block to place + */ + private static void place(Chunk chunk, int x, int y, int z, Block block) { + chunk.lockWriteLock(); + try { + chunk.setBlock(x, y, z, block); + } finally { + chunk.unlockWriteLock(); + } + } + + /** + * Reads a block of the given chunk while holding its read lock. + * + * @param chunk the chunk to read + * @param x the x coordinate inside the chunk + * @param y the y coordinate of the block + * @param z the z coordinate inside the chunk + * @return the block at the given position + */ + private static Block blockAt(Chunk chunk, int x, int y, int z) { + chunk.lockReadLock(); + try { + return chunk.getBlock(x, y, z); + } finally { + chunk.unlockReadLock(); + } + } + + /** + * Waits for the given latch and fails when it is not released in time. + * + * @param latch the latch to wait for + */ + private static void awaitStart(CountDownLatch latch) { + try { + assertTrue(latch.await(AWAIT_SECONDS, TimeUnit.SECONDS), "a worker waited too long for its barrier"); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + fail("a worker was interrupted while it waited for its barrier"); + } + } + + /** + * Waits for every given task and propagates the failure of the first broken one. + * + * @param futures the tasks to wait for + * @throws InterruptedException if the waiting thread is interrupted + * @throws ExecutionException if a task failed + */ + private static void awaitAll(List> futures) throws InterruptedException, ExecutionException { + for (Future future : futures) { + future.get(); + } + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoaderIntegrationTest.java b/src/test/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoaderIntegrationTest.java new file mode 100644 index 00000000..261f0f55 --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoaderIntegrationTest.java @@ -0,0 +1,495 @@ +package net.theevilreaper.aves.instance.anvil; + +import net.kyori.adventure.key.Key; +import net.minestom.server.instance.Chunk; +import net.minestom.server.instance.Instance; +import net.minestom.server.instance.block.Block; +import net.minestom.testing.Env; +import net.minestom.testing.extension.MicrotusExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the chunk loader against a running Minestom environment. The tests cover the round trip + * of a chunk through the region file which is the behaviour the loader exists for. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ExtendWith(MicrotusExtension.class) +class AvesAnvilLoaderIntegrationTest { + + private static final Key OVERWORLD = Key.key("minecraft:overworld"); + + @TempDir + private Path worldRoot; + + /** + * Creates a loader for the temporary world of the test. + * + * @return the created loader + */ + private AvesAnvilLoader loader() { + return new AvesAnvilLoader(this.worldRoot, OVERWORLD); + } + + @Test + void testLoadingAnAbsentChunkReturnsNull(Env env) throws IOException { + try (AvesAnvilLoader loader = loader()) { + Instance instance = env.createEmptyInstance(loader); + + assertNull(loader.loadChunk(instance, 0, 0)); + } + } + + @Test + void testTheLoaderReportsParallelSupport() throws IOException { + try (AvesAnvilLoader loader = loader()) { + assertTrue(loader.supportsParallelLoading()); + assertTrue(loader.supportsParallelSaving()); + } + } + + @Test + void testASavedChunkKeepsItsBlocks(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + Chunk chunk = instance.loadChunk(2, 3).join(); + place(chunk, 0, 40, 0, Block.STONE); + place(chunk, 5, 41, 7, Block.DIRT); + place(chunk, 15, 42, 15, Block.OAK_PLANKS); + + try (AvesAnvilLoader writer = loader()) { + writer.saveChunk(chunk); + } + + try (AvesAnvilLoader reader = loader()) { + Chunk loaded = reader.loadChunk(instance, 2, 3); + + assertNotNull(loaded); + assertEquals(Block.STONE, blockAt(loaded, 0, 40, 0)); + assertEquals(Block.DIRT, blockAt(loaded, 5, 41, 7)); + assertEquals(Block.OAK_PLANKS, blockAt(loaded, 15, 42, 15)); + assertEquals(Block.AIR, blockAt(loaded, 1, 40, 0)); + } + } + + @Test + void testTheRegionFileIsCreatedInTheDimensionDirectory(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + Chunk chunk = instance.loadChunk(0, 0).join(); + place(chunk, 0, 40, 0, Block.STONE); + + try (AvesAnvilLoader writer = loader()) { + writer.saveChunk(chunk); + } + + Path expected = this.worldRoot.resolve("dimensions/minecraft/overworld/region/r.0.0.mca"); + assertTrue(Files.exists(expected), "expected a region file at " + expected); + } + + @Test + void testABlockWithNbtSurvivesTheRoundTrip(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + Chunk chunk = instance.loadChunk(0, 0).join(); + Block sign = Block.OAK_SIGN.withNbt(net.kyori.adventure.nbt.CompoundBinaryTag.builder() + .putString("aves_marker", "kept") + .build()); + place(chunk, 3, 45, 3, sign); + + try (AvesAnvilLoader writer = loader()) { + writer.saveChunk(chunk); + } + + try (AvesAnvilLoader reader = loader()) { + Chunk loaded = reader.loadChunk(instance, 0, 0); + + assertNotNull(loaded); + Block restored = blockAt(loaded, 3, 45, 3); + assertEquals(Block.OAK_SIGN.key(), restored.key()); + assertEquals("kept", restored.nbtOrEmpty().getString("aves_marker")); + } + } + + @Test + void testBlockEntitiesAreStoredWithAbsoluteCoordinates(Env env) throws IOException { + // The format stores the position of a block entity in world coordinates. Writing chunk + // local ones still round trips through this loader, but the file would not be readable + // by the game or by any other tool. + Instance instance = env.createEmptyInstance(loader()); + Chunk chunk = instance.loadChunk(2, 3).join(); + Block sign = Block.OAK_SIGN.withNbt(net.kyori.adventure.nbt.CompoundBinaryTag.builder() + .putString("aves_marker", "kept") + .build()); + place(chunk, 5, 45, 7, sign); + + try (AvesAnvilLoader writer = loader()) { + writer.saveChunk(chunk); + } + + net.kyori.adventure.nbt.CompoundBinaryTag data = readStoredChunk(2, 3); + net.kyori.adventure.nbt.ListBinaryTag entities = + data.getList("block_entities", net.kyori.adventure.nbt.BinaryTagTypes.COMPOUND); + + assertEquals(1, entities.size()); + net.kyori.adventure.nbt.CompoundBinaryTag entity = entities.getCompound(0); + assertEquals(2 * 16 + 5, entity.getInt("x")); + assertEquals(45, entity.getInt("y")); + assertEquals(3 * 16 + 7, entity.getInt("z")); + } + + @Test + void testABlockEntityInAFarChunkIsRestoredAtItsPosition(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + Chunk chunk = instance.loadChunk(5, 9).join(); + Block sign = Block.OAK_SIGN.withNbt(net.kyori.adventure.nbt.CompoundBinaryTag.builder() + .putString("aves_marker", "far") + .build()); + place(chunk, 1, 44, 2, sign); + + try (AvesAnvilLoader writer = loader()) { + writer.saveChunk(chunk); + } + + try (AvesAnvilLoader reader = loader()) { + Chunk loaded = reader.loadChunk(instance, 5, 9); + + assertNotNull(loaded); + assertEquals("far", blockAt(loaded, 1, 44, 2).nbtOrEmpty().getString("aves_marker")); + } + } + + /** + * Reads the stored chunk data straight from the region file without using the loader. + * + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @return the stored chunk data + * @throws IOException if the chunk cannot be read + */ + private net.kyori.adventure.nbt.CompoundBinaryTag readStoredChunk(int chunkX, int chunkZ) throws IOException { + Path region = this.worldRoot.resolve("dimensions/minecraft/overworld/region") + .resolve("r." + (chunkX >> 5) + "." + (chunkZ >> 5) + ".mca"); + + try (RegionFile file = RegionFile.open(region)) { + RegionFile.RawChunk raw = file.readRaw(chunkX, chunkZ); + assertNotNull(raw); + return net.kyori.adventure.nbt.BinaryTagIO.unlimitedReader().read( + new java.io.ByteArrayInputStream(raw.decompress()), + net.kyori.adventure.nbt.BinaryTagIO.Compression.NONE + ); + } + } + + @Test + void testBlocksWithPropertiesKeepThem(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + Chunk chunk = instance.loadChunk(1, 1).join(); + Block slab = Block.OAK_SLAB.withProperty("type", "top"); + place(chunk, 4, 44, 4, slab); + + try (AvesAnvilLoader writer = loader()) { + writer.saveChunk(chunk); + } + + try (AvesAnvilLoader reader = loader()) { + Chunk loaded = reader.loadChunk(instance, 1, 1); + + assertNotNull(loaded); + assertEquals("top", blockAt(loaded, 4, 44, 4).getProperty("type")); + } + } + + @Test + void testSavingManyChunksInParallelKeepsEveryOne(Env env) throws IOException, InterruptedException, ExecutionException { + Instance instance = env.createEmptyInstance(loader()); + List chunks = new ArrayList<>(); + + for (int x = 0; x < 4; x++) { + for (int z = 0; z < 4; z++) { + Chunk chunk = instance.loadChunk(x, z).join(); + place(chunk, 0, 40, 0, Block.STONE); + place(chunk, 1, 40, 0, x + z == 0 ? Block.DIRT : Block.OAK_PLANKS); + chunks.add(chunk); + } + } + + try (AvesAnvilLoader writer = loader()) { + writer.saveChunks(chunks); + } + + try (AvesAnvilLoader reader = loader()) { + for (Chunk chunk : chunks) { + Chunk loaded = reader.loadChunk(instance, chunk.getChunkX(), chunk.getChunkZ()); + + assertNotNull(loaded, "chunk " + chunk.getChunkX() + "/" + chunk.getChunkZ() + " is missing"); + assertEquals(Block.STONE, blockAt(loaded, 0, 40, 0)); + } + } + } + + @Test + void testLoadingInParallelReturnsEveryChunk(Env env) throws IOException, InterruptedException, ExecutionException { + Instance instance = env.createEmptyInstance(loader()); + List chunks = new ArrayList<>(); + + for (int x = 0; x < 4; x++) { + Chunk chunk = instance.loadChunk(x, 0).join(); + place(chunk, 0, 40, 0, Block.STONE); + chunks.add(chunk); + } + + try (AvesAnvilLoader writer = loader()) { + writer.saveChunks(chunks); + } + + try (AvesAnvilLoader reader = loader(); + ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(); + + for (int x = 0; x < 4; x++) { + int chunkX = x; + futures.add(executor.submit(() -> reader.loadChunk(instance, chunkX, 0))); + } + for (Future future : futures) { + assertNotNull(future.get()); + } + } + } + + @Test + void testTheDiagnosticsCountTheProcessedChunks(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + Chunk chunk = instance.loadChunk(0, 0).join(); + place(chunk, 0, 40, 0, Block.STONE); + + try (AvesAnvilLoader loader = loader()) { + loader.saveChunk(chunk); + loader.loadChunk(instance, 0, 0); + + assertEquals(1, loader.diagnostics().chunksSaved()); + assertEquals(1, loader.diagnostics().chunksLoaded()); + assertEquals(0, loader.diagnostics().errors()); + } + } + + @Test + void testACorruptedChunkFailsInsteadOfLookingAbsent(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + Chunk chunk = instance.loadChunk(0, 0).join(); + place(chunk, 0, 40, 0, Block.STONE); + + try (AvesAnvilLoader writer = loader()) { + writer.saveChunk(chunk); + } + + // Overwrite the payload with bytes which are not a valid compressed chunk. Reporting this + // as an absent chunk would make the server regenerate it and overwrite the real data. + Path region = this.worldRoot.resolve("dimensions/minecraft/overworld/region/r.0.0.mca"); + byte[] bytes = Files.readAllBytes(region); + java.util.Arrays.fill(bytes, RegionConstants.HEADER_SIZE + 5, bytes.length, (byte) 0x7F); + Files.write(region, bytes); + + try (AvesAnvilLoader reader = loader()) { + // The test environment turns a reported exception into an assertion error, so the test + // asserts on the propagation itself instead of on a concrete type. What matters is that + // the call does not return null, which would make the server regenerate the chunk. + Throwable failure = assertThrows(Throwable.class, () -> reader.loadChunk(instance, 0, 0)); + + assertNotNull(failure); + assertEquals(1, reader.diagnostics().errors()); + } + } + + @Test + void testABlockHandlerSurvivesTheRoundTrip(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + Chunk chunk = instance.loadChunk(0, 0).join(); + net.minestom.server.instance.block.BlockHandler handler = + net.minestom.server.MinecraftServer.getBlockManager().getHandlerOrDummy("minecraft:sign"); + place(chunk, 6, 46, 6, Block.OAK_SIGN.withHandler(handler)); + + try (AvesAnvilLoader writer = loader()) { + writer.saveChunk(chunk); + } + + try (AvesAnvilLoader reader = loader()) { + Chunk loaded = reader.loadChunk(instance, 0, 0); + + assertNotNull(loaded); + Block restored = blockAt(loaded, 6, 46, 6); + assertNotNull(restored.handler(), "the block handler must be restored"); + assertEquals("minecraft:sign", restored.handler().getKey().asString()); + } + } + + @Test + void testTheHandlerIdIsNotKeptAsBlockNbt(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + Chunk chunk = instance.loadChunk(0, 0).join(); + net.minestom.server.instance.block.BlockHandler handler = + net.minestom.server.MinecraftServer.getBlockManager().getHandlerOrDummy("minecraft:sign"); + place(chunk, 7, 46, 7, Block.OAK_SIGN.withHandler(handler)); + + try (AvesAnvilLoader writer = loader()) { + writer.saveChunk(chunk); + } + + try (AvesAnvilLoader reader = loader()) { + Chunk loaded = reader.loadChunk(instance, 0, 0); + + assertNotNull(loaded); + // The position and the handler id belong to the file format, not to the block itself. + assertEquals("", blockAt(loaded, 7, 46, 7).nbtOrEmpty().getString("id")); + } + } + + @Test + void testUnloadingEveryChunkOfARegionClosesItsFile(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + + try (AvesAnvilLoader loader = loader()) { + Chunk first = instance.loadChunk(0, 0).join(); + Chunk second = instance.loadChunk(1, 0).join(); + place(first, 0, 40, 0, Block.STONE); + place(second, 0, 40, 0, Block.STONE); + loader.saveChunk(first); + loader.saveChunk(second); + + Chunk loadedFirst = loader.loadChunk(instance, 0, 0); + Chunk loadedSecond = loader.loadChunk(instance, 1, 0); + assertNotNull(loadedFirst); + assertNotNull(loadedSecond); + assertEquals(1, loader.openRegionCount()); + + loader.unloadChunk(loadedFirst); + assertEquals(1, loader.openRegionCount(), "the file is still used by the second chunk"); + + loader.unloadChunk(loadedSecond); + assertEquals(0, loader.openRegionCount(), "the last chunk of the region was unloaded"); + } + } + + @Test + void testAnUnloadedChunkCanBeLoadedAgain(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + + try (AvesAnvilLoader loader = loader()) { + Chunk chunk = instance.loadChunk(0, 0).join(); + place(chunk, 0, 40, 0, Block.STONE); + loader.saveChunk(chunk); + + Chunk loaded = loader.loadChunk(instance, 0, 0); + assertNotNull(loaded); + loader.unloadChunk(loaded); + + Chunk again = loader.loadChunk(instance, 0, 0); + + assertNotNull(again); + assertEquals(Block.STONE, blockAt(again, 0, 40, 0)); + } + } + + @Test + void testTheAmountOfOpenRegionFilesStaysBounded(Env env) throws IOException { + // Chunks are unloaded without telling the loader which of its region files became unused, + // so the loader has to bound the amount of open files itself instead of counting users. + Instance instance = env.createEmptyInstance(loader()); + + try (AvesAnvilLoader loader = new AvesAnvilLoader(this.worldRoot, OVERWORLD, 2)) { + for (int region = 0; region < 5; region++) { + Chunk chunk = instance.loadChunk(region * 32, 0).join(); + place(chunk, 0, 40, 0, Block.STONE); + loader.saveChunk(chunk); + } + + assertTrue(loader.openRegionCount() <= 2, "expected at most two open region files but found " + loader.openRegionCount()); + } + } + + @Test + void testAChunkStaysReadableAfterItsRegionFileWasEvicted(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + + try (AvesAnvilLoader loader = new AvesAnvilLoader(this.worldRoot, OVERWORLD, 1)) { + Chunk first = instance.loadChunk(0, 0).join(); + place(first, 0, 40, 0, Block.STONE); + loader.saveChunk(first); + + Chunk second = instance.loadChunk(64, 0).join(); + place(second, 0, 40, 0, Block.DIRT); + loader.saveChunk(second); + + // The first region file was evicted by now and has to be reopened transparently. + Chunk reloaded = loader.loadChunk(instance, 0, 0); + + assertNotNull(reloaded); + assertEquals(Block.STONE, blockAt(reloaded, 0, 40, 0)); + } + } + + @Test + void testUnloadingAForeignChunkIsIgnored(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + Chunk chunk = instance.loadChunk(9, 9).join(); + + try (AvesAnvilLoader loader = loader()) { + loader.unloadChunk(chunk); + } + } + + /** + * Places a block in the given chunk while holding its write lock. + * The block setter of a chunk requires the caller to hold that lock. + * + * @param chunk the chunk which receives the block + * @param x the x coordinate inside the chunk + * @param y the y coordinate of the block + * @param z the z coordinate inside the chunk + * @param block the block to place + */ + private static void place(Chunk chunk, int x, int y, int z, Block block) { + chunk.lockWriteLock(); + try { + chunk.setBlock(x, y, z, block); + } finally { + chunk.unlockWriteLock(); + } + } + + /** + * Reads a block of the given chunk while holding its read lock. + * The block getter of a chunk requires the caller to hold that lock. + * + * @param chunk the chunk to read + * @param x the x coordinate inside the chunk + * @param y the y coordinate of the block + * @param z the z coordinate inside the chunk + * @return the block at the given position + */ + private static Block blockAt(Chunk chunk, int x, int y, int z) { + chunk.lockReadLock(); + try { + return chunk.getBlock(x, y, z); + } finally { + chunk.unlockReadLock(); + } + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoaderLifecycleTest.java b/src/test/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoaderLifecycleTest.java new file mode 100644 index 00000000..83edf9ea --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/anvil/AvesAnvilLoaderLifecycleTest.java @@ -0,0 +1,230 @@ +package net.theevilreaper.aves.instance.anvil; + +import net.kyori.adventure.key.Key; +import net.minestom.server.MinecraftServer; +import net.minestom.server.exception.ExceptionHandler; +import net.minestom.server.instance.Chunk; +import net.minestom.server.instance.Instance; +import net.minestom.server.instance.block.Block; +import net.minestom.testing.Env; +import net.minestom.testing.extension.MicrotusExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Verifies that a closed loader stays closed. + *

+ * The loader reports parallel loading and saving as supported, so a server runs one task per chunk + * and those tasks are still in flight when the shutdown closes the loader. A task which reaches the + * region cache after that would open a file which nobody closes again, and it would write into a + * world which is already considered closed. The tests here pin the behaviour of that window down. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ExtendWith(MicrotusExtension.class) +class AvesAnvilLoaderLifecycleTest { + + private static final Key OVERWORLD = Key.key("minecraft:overworld"); + + @TempDir + private Path worldRoot; + + /** + * Creates a loader for the temporary world of the test. + * + * @return the created loader + */ + private AvesAnvilLoader loader() { + return new AvesAnvilLoader(this.worldRoot, OVERWORLD); + } + + /** + * Writes a single chunk into the temporary world so the load path has something to read. + * + * @param instance the instance which owns the chunk + * @return the written chunk + * @throws IOException if the chunk cannot be written + */ + private Chunk storeChunk(Instance instance) throws IOException { + Chunk chunk = instance.loadChunk(0, 0).join(); + place(chunk, 0, 40, 0, Block.STONE); + + try (AvesAnvilLoader writer = loader()) { + writer.saveChunk(chunk); + } + return chunk; + } + + @Test + void testLoadingAfterCloseIsRejected(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + storeChunk(instance); + + AvesAnvilLoader loader = loader(); + loader.close(); + + // Returning the chunk would use a loader which released its files, and returning null would + // make the server generate a replacement which overwrites the stored chunk on the next save. + assertThrows(IllegalStateException.class, () -> loader.loadChunk(instance, 0, 0)); + } + + @Test + void testLoadingAfterCloseOpensNoRegionFile(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + storeChunk(instance); + + AvesAnvilLoader loader = loader(); + loader.close(); + + assertThrows(IllegalStateException.class, () -> loader.loadChunk(instance, 0, 0)); + assertEquals( + 0, loader.openRegionCount(), + "a closed loader may not hold a region file because nothing closes it again" + ); + } + + @Test + void testSavingAfterCloseIsRejected(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + Chunk chunk = storeChunk(instance); + + AvesAnvilLoader loader = loader(); + loader.close(); + + // Swallowing the call would write into a world which is already considered closed and leak + // the region file it opened for that write. + assertThrows(IllegalStateException.class, () -> loader.saveChunk(chunk)); + assertEquals(0, loader.openRegionCount(), "a closed loader may not hold a region file"); + } + + @Test + void testAClosedLoaderCanBeClosedAgain(Env env) throws IOException { + Instance instance = env.createEmptyInstance(loader()); + storeChunk(instance); + + AvesAnvilLoader loader = loader(); + Chunk loaded = loader.loadChunk(instance, 0, 0); + + assertNotNull(loaded); + loader.close(); + loader.close(); + + assertEquals(0, loader.openRegionCount()); + } + + @Test + void testClosingWhileLoadsAreRunningLeavesNoOpenRegionFile(Env env) throws IOException, InterruptedException, ExecutionException { + // A loader is closed while its tasks are still running, because the loader reports parallel + // loading as supported and therefore receives one task per chunk. A task which reaches the + // region cache after the close would open a file which nothing closes again, and it would + // find the cache emptied right under it. A load which is refused is fine, a load which + // fails on a file the loader itself closed is not, and a load which reports the chunk as + // absent would make the server overwrite it. + int regionCount = 8; + int rounds = 40; + Instance instance = env.createEmptyInstance(loader()); + List chunks = new ArrayList<>(regionCount); + + for (int region = 0; region < regionCount; region++) { + Chunk chunk = instance.loadChunk(region * 32, 0).join(); + place(chunk, 0, 40, 0, Block.STONE); + chunks.add(chunk); + } + + try (AvesAnvilLoader writer = loader()) { + writer.saveChunks(chunks); + } + + List failures = new CopyOnWriteArrayList<>(); + CountDownLatch running = new CountDownLatch(regionCount); + AvesAnvilLoader loader = loader(); + ExceptionHandler previous = MinecraftServer.getExceptionManager().getExceptionHandler(); + MinecraftServer.getExceptionManager().setExceptionHandler(ignored -> { + }); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(chunks.size()); + + for (Chunk chunk : chunks) { + futures.add(executor.submit(() -> { + running.countDown(); + + for (int round = 0; round < rounds; round++) { + try { + Chunk loaded = loader.loadChunk(instance, chunk.getChunkX(), chunk.getChunkZ()); + + if (loaded == null) { + failures.add(new IllegalStateException( + "the chunk " + chunk.getChunkX() + "/" + chunk.getChunkZ() + " was reported as absent" + )); + } + } catch (IllegalStateException expected) { + // A loader which is closed refuses further work, which is the contract. + return null; + } catch (Throwable failure) { + failures.add(failure); + } + } + return null; + })); + } + assertTrue(running.await(60L, TimeUnit.SECONDS), "the workers did not start in time"); + loader.close(); + + for (Future future : futures) { + future.get(); + } + } finally { + MinecraftServer.getExceptionManager().setExceptionHandler(previous); + } + + if (!failures.isEmpty()) { + fail("a load may only be refused, never fail, while the loader closes but " + failures.size() + + " of them failed, the first one was: " + failures.getFirst(), failures.getFirst()); + } + assertEquals( + 0, loader.openRegionCount(), + "a task which ran into the close may not leave a region file behind because nothing closes it again" + ); + } + + /** + * Places a block in the given chunk while holding its write lock. + * + * @param chunk the chunk which receives the block + * @param x the x coordinate inside the chunk + * @param y the y coordinate of the block + * @param z the z coordinate inside the chunk + * @param block the block to place + */ + private static void place(Chunk chunk, int x, int y, int z, Block block) { + chunk.lockWriteLock(); + try { + chunk.setBlock(x, y, z, block); + } finally { + chunk.unlockWriteLock(); + } + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/anvil/BitPackerTest.java b/src/test/java/net/theevilreaper/aves/instance/anvil/BitPackerTest.java new file mode 100644 index 00000000..ec2a6d14 --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/anvil/BitPackerTest.java @@ -0,0 +1,122 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.random.RandomGenerator; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Tests the bit packing which converts palette indices into the packed long array + * representation used by the Anvil format and back. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class BitPackerTest { + + @ParameterizedTest + @CsvSource({"1, 1", "2, 1", "3, 2", "4, 2", "5, 3", "8, 3", "9, 4", "16, 4", "17, 5"}) + void testBitsPerEntryGrowsWithThePaletteSize(int paletteSize, int expectedBits) { + assertEquals(expectedBits, BitPacker.bitsPerEntry(paletteSize, 1)); + } + + @Test + void testBitsPerEntryRespectsTheMinimum() { + assertEquals(4, BitPacker.bitsPerEntry(2, 4)); + } + + @ParameterizedTest + @ValueSource(ints = {0, -1}) + void testBitsPerEntryRejectsAnEmptyPalette(int paletteSize) { + assertThrows(IllegalArgumentException.class, () -> BitPacker.bitsPerEntry(paletteSize, 1)); + } + + @ParameterizedTest + @CsvSource({"4096, 4, 256", "4096, 5, 342", "4096, 15, 1024", "64, 1, 1", "64, 3, 4"}) + void testExpectedLongCountFollowsTheEntriesPerLong(int entryCount, int bitsPerEntry, int expected) { + assertEquals(expected, BitPacker.expectedLongCount(entryCount, bitsPerEntry)); + } + + @Test + void testPackWritesEntriesWithoutSpanningALongBoundary() { + // With five bits per entry twelve entries fit into a long and four bits stay unused. + // The last entry of a long must not bleed into the next one. + int[] values = new int[64]; + values[11] = 0b11111; + + long[] packed = BitPacker.pack(values, 5); + + assertEquals(6, packed.length); + assertEquals(0b11111L, (packed[0] >>> 55) & 0b11111L); + assertEquals(0L, packed[1], "the entry must stay inside the first long"); + } + + @Test + void testPackStartsANewLongAfterTheEntriesPerLongAreExhausted() { + int[] values = new int[64]; + values[12] = 1; + + long[] packed = BitPacker.pack(values, 5); + + assertEquals(0L, packed[0]); + assertEquals(1L, packed[1] & 0b11111L); + } + + @Test + void testPackLeavesThePaddingBitsOfEachLongEmpty() { + int[] values = new int[64]; + java.util.Arrays.fill(values, 0b11111); + + long[] packed = BitPacker.pack(values, 5); + + for (long entry : packed) { + assertEquals(0L, entry >>> 60, "the upper padding bits must stay empty"); + } + } + + @ParameterizedTest + @ValueSource(ints = {1, 2, 3, 4, 5, 6, 7, 8, 12, 15}) + void testUnpackReversesPackForRandomData(int bitsPerEntry) { + RandomGenerator random = RandomGenerator.getDefault(); + int[] values = new int[4096]; + int bound = 1 << bitsPerEntry; + + for (int i = 0; i < values.length; i++) { + values[i] = random.nextInt(bound); + } + + long[] packed = BitPacker.pack(values, bitsPerEntry); + + assertEquals(BitPacker.expectedLongCount(values.length, bitsPerEntry), packed.length); + assertArrayEquals(values, BitPacker.unpack(packed, values.length, bitsPerEntry)); + } + + @Test + void testUnpackRejectsATooShortArray() { + long[] packed = new long[1]; + + assertThrows(IllegalArgumentException.class, () -> BitPacker.unpack(packed, 4096, 4)); + } + + @Test + void testResolveBitsPerEntryDerivesTheValueFromTheArrayLength() { + assertEquals(5, BitPacker.resolveBitsPerEntry(342, 4096, 4)); + } + + @Test + void testResolveBitsPerEntryKeepsTheExpectedValueWhenTheLengthMatches() { + assertEquals(4, BitPacker.resolveBitsPerEntry(256, 4096, 4)); + } + + @Test + void testResolveBitsPerEntryReturnsZeroForAnUnmatchableLength() { + assertEquals(0, BitPacker.resolveBitsPerEntry(7, 4096, 4)); + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/anvil/ChunkCompressionTest.java b/src/test/java/net/theevilreaper/aves/instance/anvil/ChunkCompressionTest.java new file mode 100644 index 00000000..bd216283 --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/anvil/ChunkCompressionTest.java @@ -0,0 +1,121 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the compression schemes which the Anvil format supports for a chunk payload. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class ChunkCompressionTest { + + private static final byte[] PAYLOAD = "a chunk payload which repeats a chunk payload".getBytes(StandardCharsets.UTF_8); + + @ParameterizedTest + @EnumSource(ChunkCompression.class) + void testCompressAndDecompressAreInverse(ChunkCompression compression) throws IOException { + byte[] compressed = compression.compress(PAYLOAD); + + assertArrayEquals(PAYLOAD, compression.decompress(compressed)); + } + + @ParameterizedTest + @EnumSource(ChunkCompression.class) + void testEveryCompressionExposesItsFormatIdentifier(ChunkCompression compression) throws IOException { + assertEquals(compression, ChunkCompression.fromId(compression.id())); + } + + @Test + void testGzipIsIdentifiedByOne() throws IOException { + assertEquals(ChunkCompression.GZIP, ChunkCompression.fromId(1)); + } + + @Test + void testZlibIsIdentifiedByTwo() throws IOException { + assertEquals(ChunkCompression.ZLIB, ChunkCompression.fromId(2)); + } + + @Test + void testNoneIsIdentifiedByThree() throws IOException { + assertEquals(ChunkCompression.NONE, ChunkCompression.fromId(3)); + } + + @ParameterizedTest + @ValueSource(ints = {0, 4, 5, 127}) + void testUnsupportedSchemesAreRejectedWithTheirIdentifier(int id) { + IOException exception = assertThrows(IOException.class, () -> ChunkCompression.fromId(id)); + + assertTrue(exception.getMessage().contains(String.valueOf(id))); + } + + @Test + void testTheExternalFlagIsDetected() { + assertTrue(ChunkCompression.isExternal(2 | ChunkCompression.EXTERNAL_FLAG)); + assertFalse(ChunkCompression.isExternal(2)); + } + + @Test + void testTheExternalFlagIsStrippedBeforeResolving() throws IOException { + assertEquals(ChunkCompression.ZLIB, ChunkCompression.fromId(2 | ChunkCompression.EXTERNAL_FLAG)); + } + + @Test + void testNoneKeepsThePayloadUntouched() throws IOException { + assertArrayEquals(PAYLOAD, ChunkCompression.NONE.compress(PAYLOAD)); + } + + @Test + void testZlibActuallyShrinksARepetitivePayload() throws IOException { + byte[] repetitive = new byte[4096]; + + assertTrue(ChunkCompression.ZLIB.compress(repetitive).length < repetitive.length); + } + + @Test + void testTheCompressionLevelCanBeChosen() throws IOException { + byte[] payload = new byte[64 * 1024]; + new java.util.Random(7).nextBytes(payload); + // Half the array is compressible so the level actually has an effect. + java.util.Arrays.fill(payload, 0, payload.length / 2, (byte) 7); + + byte[] fast = ChunkCompression.ZLIB.compress(payload, ChunkCompression.FASTEST_LEVEL); + byte[] balanced = ChunkCompression.ZLIB.compress(payload, ChunkCompression.DEFAULT_LEVEL); + + assertArrayEquals(payload, ChunkCompression.ZLIB.decompress(fast)); + assertArrayEquals(payload, ChunkCompression.ZLIB.decompress(balanced)); + assertTrue(balanced.length <= fast.length, "a higher level must not produce a larger result"); + } + + @Test + void testEveryLevelRoundTrips() throws IOException { + for (int level = ChunkCompression.FASTEST_LEVEL; level <= ChunkCompression.SMALLEST_LEVEL; level++) { + assertArrayEquals(PAYLOAD, ChunkCompression.ZLIB.decompress(ChunkCompression.ZLIB.compress(PAYLOAD, level)), + "level " + level + " has to round trip"); + } + } + + @Test + void testAnInvalidLevelIsRejected() { + assertThrows(IllegalArgumentException.class, () -> ChunkCompression.ZLIB.compress(PAYLOAD, 0)); + assertThrows(IllegalArgumentException.class, () -> ChunkCompression.ZLIB.compress(PAYLOAD, 10)); + } + + @Test + void testTheLevelIsIgnoredWithoutCompression() throws IOException { + assertArrayEquals(PAYLOAD, ChunkCompression.NONE.compress(PAYLOAD, ChunkCompression.SMALLEST_LEVEL)); + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/anvil/NbtReadsTest.java b/src/test/java/net/theevilreaper/aves/instance/anvil/NbtReadsTest.java new file mode 100644 index 00000000..27772013 --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/anvil/NbtReadsTest.java @@ -0,0 +1,143 @@ +package net.theevilreaper.aves.instance.anvil; + +import net.kyori.adventure.nbt.BinaryTagTypes; +import net.kyori.adventure.nbt.CompoundBinaryTag; +import net.kyori.adventure.nbt.IntBinaryTag; +import net.kyori.adventure.nbt.ListBinaryTag; +import net.kyori.adventure.nbt.LongArrayBinaryTag; +import net.kyori.adventure.nbt.StringBinaryTag; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the strict access facade around Adventure NBT. The facade has to turn the silent + * defaults of the library into explicit errors and must not rely on the broken iterators + * of the array tags. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class NbtReadsTest { + + @Test + void testTheArrayIteratorOfAdventureSkipsTheLastEntry() { + // This documents the library defect the facade has to work around. A for each loop over + // a long array tag silently drops the last entry which would corrupt every chunk. + LongArrayBinaryTag tag = LongArrayBinaryTag.longArrayBinaryTag(1L, 2L, 3L, 4L); + int visited = 0; + + for (long ignored : tag) { + visited++; + } + + assertEquals(4, tag.size()); + assertEquals(3, visited, "adventure-nbt 5.1.1 is expected to skip the last entry here"); + } + + @Test + void testLongArrayReadsEveryEntry() throws IOException { + CompoundBinaryTag compound = CompoundBinaryTag.builder() + .put("data", LongArrayBinaryTag.longArrayBinaryTag(1L, 2L, 3L, 4L)) + .build(); + + assertArrayEquals(new long[]{1L, 2L, 3L, 4L}, NbtReads.longArray(compound, "data")); + } + + @Test + void testLongArrayFailsForAMissingKey() { + CompoundBinaryTag compound = CompoundBinaryTag.empty(); + + IOException exception = assertThrows(IOException.class, () -> NbtReads.longArray(compound, "data")); + + assertTrue(exception.getMessage().contains("data")); + } + + @Test + void testLongArrayFailsForAWrongType() { + CompoundBinaryTag compound = CompoundBinaryTag.builder().put("data", StringBinaryTag.stringBinaryTag("nope")).build(); + + assertThrows(IOException.class, () -> NbtReads.longArray(compound, "data")); + } + + @Test + void testCompoundReturnsTheNestedCompound() throws IOException { + CompoundBinaryTag nested = CompoundBinaryTag.builder().putInt("value", 7).build(); + CompoundBinaryTag compound = CompoundBinaryTag.builder().put("nested", nested).build(); + + assertSame(nested, NbtReads.compound(compound, "nested")); + } + + @Test + void testCompoundFailsForAMissingKey() { + assertThrows(IOException.class, () -> NbtReads.compound(CompoundBinaryTag.empty(), "nested")); + } + + @Test + void testOptionalCompoundReturnsNullForAMissingKey() { + assertEquals(null, NbtReads.optionalCompound(CompoundBinaryTag.empty(), "nested")); + } + + @Test + void testListReturnsTheTypedList() throws IOException { + ListBinaryTag list = ListBinaryTag.builder(BinaryTagTypes.COMPOUND) + .add(CompoundBinaryTag.empty()) + .build(); + CompoundBinaryTag compound = CompoundBinaryTag.builder().put("sections", list).build(); + + assertEquals(1, NbtReads.list(compound, "sections", BinaryTagTypes.COMPOUND).size()); + } + + @Test + void testListFailsForAWrongElementType() { + ListBinaryTag list = ListBinaryTag.builder(BinaryTagTypes.INT).add(IntBinaryTag.intBinaryTag(1)).build(); + CompoundBinaryTag compound = CompoundBinaryTag.builder().put("sections", list).build(); + + assertThrows(IOException.class, () -> NbtReads.list(compound, "sections", BinaryTagTypes.COMPOUND)); + } + + @Test + void testOptionalListReturnsAnEmptyListForAMissingKey() { + assertEquals(0, NbtReads.optionalList(CompoundBinaryTag.empty(), "sections", BinaryTagTypes.COMPOUND).size()); + } + + @Test + void testStringReturnsTheValue() throws IOException { + CompoundBinaryTag compound = CompoundBinaryTag.builder().putString("Name", "minecraft:stone").build(); + + assertEquals("minecraft:stone", NbtReads.string(compound, "Name")); + } + + @Test + void testStringFailsForANumericValue() { + CompoundBinaryTag compound = CompoundBinaryTag.builder().putInt("Name", 3).build(); + + assertThrows(IOException.class, () -> NbtReads.string(compound, "Name")); + } + + @Test + void testIntReturnsTheValue() throws IOException { + CompoundBinaryTag compound = CompoundBinaryTag.builder().putInt("DataVersion", 4790).build(); + + assertEquals(4790, NbtReads.integer(compound, "DataVersion")); + } + + @Test + void testIntAcceptsANarrowerNumericType() throws IOException { + CompoundBinaryTag compound = CompoundBinaryTag.builder().putByte("Y", (byte) -4).build(); + + assertEquals(-4, NbtReads.integer(compound, "Y")); + } + + @Test + void testIntFailsForAMissingKey() { + assertThrows(IOException.class, () -> NbtReads.integer(CompoundBinaryTag.empty(), "DataVersion")); + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/anvil/PaletteDataTest.java b/src/test/java/net/theevilreaper/aves/instance/anvil/PaletteDataTest.java new file mode 100644 index 00000000..6ec279ee --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/anvil/PaletteDataTest.java @@ -0,0 +1,235 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.util.random.RandomGenerator; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the palette representation which the codec uses between the region file and the + * palettes of Minestom. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class PaletteDataTest { + + private static final int BLOCK_ENTRIES = 4096; + private static final int BLOCK_MIN_BITS = 4; + + @Test + void testASingleValueNeedsNoPackedData() { + PaletteData data = PaletteData.single(7, BLOCK_ENTRIES); + + assertTrue(data.isSingleValue()); + assertEquals(7, data.singleValue()); + assertNull(data.packed()); + assertEquals(BLOCK_ENTRIES, data.entryCount()); + } + + @Test + void testEncodingAHomogeneousSectionCollapsesToASingleValue() { + int[] values = new int[BLOCK_ENTRIES]; + java.util.Arrays.fill(values, 42); + + PaletteData data = PaletteData.encode(values, BLOCK_MIN_BITS); + + assertTrue(data.isSingleValue()); + assertEquals(42, data.singleValue()); + } + + @Test + void testEncodingKeepsEveryDistinctValue() throws IOException { + int[] values = new int[BLOCK_ENTRIES]; + + for (int i = 0; i < values.length; i++) { + values[i] = i % 5; + } + + PaletteData data = PaletteData.encode(values, BLOCK_MIN_BITS); + + assertFalse(data.isSingleValue()); + assertEquals(5, data.palette().length); + assertArrayEquals(values, data.unpack()); + } + + @Test + void testEncodingRespectsTheMinimumBitsOfTheType() { + int[] values = new int[BLOCK_ENTRIES]; + values[0] = 1; + + PaletteData data = PaletteData.encode(values, BLOCK_MIN_BITS); + + assertEquals(BLOCK_MIN_BITS, data.bitsPerEntry()); + } + + @Test + void testEncodingBiomesUsesASmallerMinimum() { + int[] values = new int[64]; + values[0] = 1; + + PaletteData data = PaletteData.encode(values, 1); + + assertEquals(1, data.bitsPerEntry()); + } + + @ParameterizedTest + @ValueSource(ints = {2, 3, 16, 17, 200, 4096}) + void testEncodeAndUnpackAreInverseForRandomData(int distinctValues) throws IOException { + RandomGenerator random = RandomGenerator.getDefault(); + int[] values = new int[BLOCK_ENTRIES]; + + for (int i = 0; i < values.length; i++) { + values[i] = random.nextInt(distinctValues); + } + for (int i = 0; i < distinctValues; i++) { + values[i % values.length] = i; + } + + PaletteData data = PaletteData.encode(values, BLOCK_MIN_BITS); + + assertArrayEquals(values, data.unpack()); + } + + @Test + void testReadingAcceptsDataWhichMatchesTheExpectedBitCount() throws IOException { + int[] palette = {10, 20, 30}; + long[] packed = BitPacker.pack(new int[BLOCK_ENTRIES], BLOCK_MIN_BITS); + + PaletteData data = PaletteData.read(palette, packed, BLOCK_ENTRIES, BLOCK_MIN_BITS); + + assertEquals(BLOCK_MIN_BITS, data.bitsPerEntry()); + assertSame(packed, data.packed()); + } + + @Test + void testReadingRecoversABitCountWhichIsLargerThanTheMinimum() throws IOException { + // A foreign writer may use more bits than the palette size requires. Minestom derives the + // bit count from the palette length alone and would decode this data incorrectly. + int[] palette = {10, 20, 30}; + long[] packed = BitPacker.pack(new int[BLOCK_ENTRIES], 6); + + PaletteData data = PaletteData.read(palette, packed, BLOCK_ENTRIES, BLOCK_MIN_BITS); + + assertEquals(6, data.bitsPerEntry()); + } + + @Test + void testReadingRejectsAnUnmatchableLength() { + int[] palette = {10, 20}; + long[] packed = new long[7]; + + assertThrows(IOException.class, () -> PaletteData.read(palette, packed, BLOCK_ENTRIES, BLOCK_MIN_BITS)); + } + + @Test + void testReadingASinglePaletteEntryWithoutDataYieldsASingleValue() throws IOException { + PaletteData data = PaletteData.read(new int[]{99}, null, BLOCK_ENTRIES, BLOCK_MIN_BITS); + + assertTrue(data.isSingleValue()); + assertEquals(99, data.singleValue()); + } + + @Test + void testReadingRejectsAnEmptyPalette() { + assertThrows(IOException.class, () -> PaletteData.read(new int[0], null, BLOCK_ENTRIES, BLOCK_MIN_BITS)); + } + + @Test + void testUnpackResolvesEveryEntryThroughThePalette() throws IOException { + int[] palette = {100, 200, 300, 400}; + int[] indices = new int[BLOCK_ENTRIES]; + indices[0] = 3; + indices[1] = 1; + long[] packed = BitPacker.pack(indices, BLOCK_MIN_BITS); + + PaletteData data = PaletteData.read(palette, packed, BLOCK_ENTRIES, BLOCK_MIN_BITS); + int[] values = data.unpack(); + + assertEquals(400, values[0]); + assertEquals(200, values[1]); + assertEquals(100, values[2]); + } + + @Test + void testUnpackOfASingleValueFillsEveryEntry() throws IOException { + int[] values = PaletteData.single(5, 64).unpack(); + + assertEquals(64, values.length); + assertArrayEquals(new int[64], subtract(values, 5)); + } + + @Test + void testReadingRejectsAPaletteIndexOutsideOfThePalette() throws IOException { + int[] indices = new int[BLOCK_ENTRIES]; + indices[5] = 3; + long[] packed = BitPacker.pack(indices, BLOCK_MIN_BITS); + PaletteData data = PaletteData.read(new int[]{1, 2}, packed, BLOCK_ENTRIES, BLOCK_MIN_BITS); + + assertThrows(IOException.class, data::unpack); + } + + /** + * Subtracts the given amount from every entry of the array. + * + * @param values the values to reduce + * @param amount the amount to subtract + * @return the reduced values + */ + private static int[] subtract(int[] values, int amount) { + int[] result = new int[values.length]; + + for (int i = 0; i < values.length; i++) { + result[i] = values[i] - amount; + } + return result; + } + + @Test + void testEncodingAUniformSectionTouchesNoMap() { + // A section of one repeated state is the common case: air, stone, or water fill whole + // sections of a world. Building a palette map over 4096 identical entries to then collapse + // it again is pure waste, so the uniform case has to be recognised before that happens. + int[] values = new int[BLOCK_ENTRIES]; + java.util.Arrays.fill(values, 77); + + PaletteData data = PaletteData.encode(values, BLOCK_MIN_BITS); + + assertTrue(data.isSingleValue()); + assertEquals(77, data.singleValue()); + assertEquals(1, data.palette().length); + assertNull(data.packed()); + } + + @Test + void testEncodingStillHandlesASingleDifferingEntry() { + // The shortcut must not swallow a section that is uniform except for one block. + int[] values = new int[BLOCK_ENTRIES]; + java.util.Arrays.fill(values, 5); + values[BLOCK_ENTRIES - 1] = 6; + + PaletteData data = PaletteData.encode(values, BLOCK_MIN_BITS); + + assertFalse(data.isSingleValue()); + assertEquals(2, data.palette().length); + } + + @Test + void testEncodingAnEmptySectionIsUniform() { + PaletteData data = PaletteData.encode(new int[BLOCK_ENTRIES], BLOCK_MIN_BITS); + + assertTrue(data.isSingleValue()); + assertEquals(0, data.singleValue()); + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/anvil/RegionFileConcurrencyTest.java b/src/test/java/net/theevilreaper/aves/instance/anvil/RegionFileConcurrencyTest.java new file mode 100644 index 00000000..68612927 --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/anvil/RegionFileConcurrencyTest.java @@ -0,0 +1,1033 @@ +package net.theevilreaper.aves.instance.anvil; + +import net.theevilreaper.aves.FileTestBase; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Stresses the thread safety of the region file container. + *

+ * The whole design claim of {@link RegionFile} is that a read never takes a lock and still only ever + * returns a state which really existed, while a writer guards no more than the sector allocation, + * the header entry and the switch between the two storage locations of a chunk. Every test in this + * class is built so that it fails when that claim breaks: the payloads carry a marker byte in every + * single byte, the sector table of the finished file is checked for overlapping ranges, and the file + * is reopened afterwards so the header has to describe a layout which can be rebuilt. + *

+ *

+ * A test which merely starts many threads and asserts that nothing was thrown would pass on a + * broken implementation, so none of the tests here stop at that. Each of them names the corruption + * it detects in its own comment. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class RegionFileConcurrencyTest extends FileTestBase { + + /** + * The time a latch is waited for before the test is considered stuck. + */ + private static final long AWAIT_SECONDS = 60L; + + /** + * The amount of chunks which are written concurrently in the disjointness test. + */ + private static final int DISJOINT_CHUNK_COUNT = 64; + + /** + * The amount of chunks the torn read test rewrites while readers hammer the file. + */ + private static final int TORN_CHUNK_COUNT = 8; + + /** + * The amount of threads which read while the torn read test rewrites the chunks. + */ + private static final int TORN_READER_COUNT = 8; + + /** + * The sector count of every version the torn read test writes. + *

+ * The values are chosen so the allocator can never hand out a sector twice. All eight version + * zero payloads occupy eight sectors in total, which is less than the nine sectors a single + * version one payload needs, and version zero plus version one occupy eighty sectors in total, + * which is less than the eighty one sectors a single version two payload needs. Together with + * the barrier which holds every writer until the last one finished its version one, a freed + * range can therefore never satisfy a later allocation, not even when every freed range is + * adjacent and merges into one gap. + *

+ *

+ * That property is what lets this test compute the final size of the file down to the byte and + * assert it. It is not a workaround: a reader which observes a recycled range has to see a + * single consistent version just as well, which + * {@link #testReadersNeverObserveASectorWhichWasRecycledWhileTheyRead()} drives on purpose. + * Keeping the two apart only means that a failure here names the torn read and a failure there + * names the recycled range. + *

+ */ + private static final int[] TORN_SECTORS = {1, 9, 81}; + + /** + * The amount of sectors the file of the torn read test spans once every version was written. + * The header occupies two sectors, version zero eight, version one seventy two and version two + * six hundred and forty eight. + */ + private static final int TORN_TOTAL_SECTORS = 2 + 8 + 72 + 648; + + /** + * The payload sizes a single chunk cycles through in the grow and shrink test. + * The last entry crosses the limit of {@link RegionConstants#MAX_SECTORS_PER_CHUNK} sectors and + * therefore moves the chunk into an external file. + */ + private static final int[] CYCLE_SIZES = { + RegionConstants.SECTOR_SIZE - 5, + RegionConstants.SECTOR_SIZE * 3 - 5, + RegionConstants.SECTOR_SIZE * 2 - 5, + RegionConstants.MAX_SECTORS_PER_CHUNK * RegionConstants.SECTOR_SIZE + 1 + }; + + /** + * The amount of times a scenario which depends on an interleaving is repeated. + * A single run of such a scenario does not always hit the window a broken implementation opens, + * so the scenario is repeated on a fresh file until a defect is practically certain to show. + */ + private static final int ATTEMPTS = 4; + + /** + * The sector count of the large version the recycling test writes. + * The value is the largest one a location entry can address, which makes the payload read of a + * reader as long as the format allows and therefore widens the window a recycled range needs. + */ + private static final int RECYCLE_LARGE_SECTORS = RegionConstants.MAX_SECTORS_PER_CHUNK; + + /** + * The sector count of the small version the recycling test writes. + * Shrinking the chunk to a single sector is what frees the large range in the first place. + */ + private static final int RECYCLE_SMALL_SECTORS = 1; + + /** + * The byte every large version of the observed chunk of the recycling test carries. + */ + private static final byte RECYCLE_LARGE_MARKER = (byte) 0x51; + + /** + * The byte every small version of the observed chunk of the recycling test carries. + */ + private static final byte RECYCLE_SMALL_MARKER = (byte) 0x52; + + /** + * The amount of threads which compete for the freed range in the recycling test. + * Every one of them writes exactly as many sectors as the observed chunk frees, so the first fit + * strategy of the allocator hands the freed range to one of them. + */ + private static final int RECYCLE_FILLER_COUNT = 4; + + /** + * The amount of threads which read the observed chunk of the recycling test. + */ + private static final int RECYCLE_READER_COUNT = 8; + + /** + * The amount of versions every writer of the recycling test produces. + */ + private static final int RECYCLE_ROUNDS = 60; + + /** + * The amount of times the recycling scenario is repeated on a fresh file. + */ + private static final int RECYCLE_ATTEMPTS = 2; + + /** + * The length of the payload which does not fit into a location entry and therefore lives in an + * external file next to the region file. + */ + private static final int EXTERNAL_PAYLOAD_LENGTH = RegionConstants.MAX_SECTORS_PER_CHUNK * RegionConstants.SECTOR_SIZE + 1; + + /** + * The length of the payload which is small enough to stay inside the region file. + */ + private static final int INLINE_PAYLOAD_LENGTH = RegionConstants.SECTOR_SIZE - 5; + + /** + * The byte the external payload of the storage switch test carries. + */ + private static final byte EXTERNAL_MARKER = (byte) 0x61; + + /** + * The byte the inline payload of the storage switch test carries. + */ + private static final byte INLINE_MARKER = (byte) 0x62; + + /** + * The amount of versions every writer of the storage switch test produces. + */ + private static final int SWITCH_ROUNDS = 40; + + /** + * The amount of threads which read the chunk of the storage switch test. + */ + private static final int SWITCH_READER_COUNT = 4; + + /** + * Creates the path of a region file of the given attempt inside the temporary directory. + * + * @param attempt the index of the attempt the file belongs to + * @return the path of the region file + */ + private Path regionPath(int attempt) { + return this.tempDir.resolve("r.0." + attempt + ".mca"); + } + + @Test + void testConcurrentWritesToDistinctChunksKeepEverySectorRangeDisjoint() throws IOException, InterruptedException, ExecutionException { + for (int attempt = 0; attempt < ATTEMPTS; attempt++) { + writeDistinctChunksConcurrently(regionPath(attempt)); + } + } + + /** + * Writes every chunk of a region file from its own thread and verifies the result. + * + * @param path the path of the region file to work on + * @throws IOException if the region file cannot be used + * @throws InterruptedException if the waiting thread is interrupted + * @throws ExecutionException if a writer failed + */ + private void writeDistinctChunksConcurrently(Path path) throws IOException, InterruptedException, ExecutionException { + // A lost update inside the sector allocator hands the same sectors to two chunks. The + // payloads then overwrite each other, which the marker bytes expose, and the location table + // ends up with two entries pointing into the same range, which the disjointness check + // exposes even when the payload check happens to survive. + List payloads = new ArrayList<>(DISJOINT_CHUNK_COUNT); + + for (int index = 0; index < DISJOINT_CHUNK_COUNT; index++) { + payloads.add(marked((index % 3 + 1) * RegionConstants.SECTOR_SIZE - 5, (byte) (index + 1))); + } + + CountDownLatch start = new CountDownLatch(1); + + try (RegionFile region = RegionFile.open(path); + ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(DISJOINT_CHUNK_COUNT); + + for (int index = 0; index < DISJOINT_CHUNK_COUNT; index++) { + int chunk = index; + futures.add(executor.submit(() -> { + awaitStart(start); + region.writeRaw(chunk % 32, chunk / 32, ChunkCompression.ZLIB, payloads.get(chunk)); + return null; + })); + } + start.countDown(); + awaitAll(futures); + + for (int index = 0; index < DISJOINT_CHUNK_COUNT; index++) { + assertArrayEquals(payloads.get(index), read(region, index % 32, index / 32), "chunk " + index + " was corrupted"); + } + } + + assertSectorTableIsDisjoint(path); + + try (RegionFile reopened = RegionFile.open(path)) { + for (int index = 0; index < DISJOINT_CHUNK_COUNT; index++) { + assertArrayEquals(payloads.get(index), read(reopened, index % 32, index / 32), "chunk " + index + " did not survive the reopen"); + } + } + } + + @Test + void testConcurrentReadersNeverObserveATornPayload() throws IOException, InterruptedException, ExecutionException { + for (int attempt = 0; attempt < ATTEMPTS; attempt++) { + readWhileTheChunksAreRewritten(regionPath(attempt)); + } + } + + /** + * Rewrites every chunk of a region file twice while readers keep reading it. + * + * @param path the path of the region file to work on + * @throws IOException if the region file cannot be used + * @throws InterruptedException if the waiting thread is interrupted + * @throws ExecutionException if a worker failed + */ + private void readWhileTheChunksAreRewritten(Path path) throws IOException, InterruptedException, ExecutionException { + // Every byte of a payload encodes the chunk and the version it belongs to. A reader which + // observes a mix of two versions therefore sees two different byte values in one payload, + // which is exactly what an in place update of a chunk would produce. The readers are + // released before the writers so every one of them is guaranteed to observe the old version + // at least once, which proves that the test really reads across the transition. + Queue failures = new ConcurrentLinkedQueue<>(); + AtomicIntegerArray observations = new AtomicIntegerArray(TORN_SECTORS.length); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch warmedUp = new CountDownLatch(TORN_READER_COUNT); + CountDownLatch grown = new CountDownLatch(TORN_CHUNK_COUNT); + CountDownLatch written = new CountDownLatch(TORN_CHUNK_COUNT); + + try (RegionFile region = RegionFile.open(path); + ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + for (int chunk = 0; chunk < TORN_CHUNK_COUNT; chunk++) { + region.writeRaw(chunk, 0, ChunkCompression.ZLIB, tornPayload(chunk, 0)); + } + + List> futures = new ArrayList<>(TORN_READER_COUNT + TORN_CHUNK_COUNT); + + for (int reader = 0; reader < TORN_READER_COUNT; reader++) { + futures.add(executor.submit(() -> { + awaitStart(start); + + // The first pass has to finish before the writers are released, so it is + // performed outside of the loop and reports its end even when it failed. + try { + for (int chunk = 0; chunk < TORN_CHUNK_COUNT; chunk++) { + inspectTornRead(region.readRaw(chunk, 0), chunk, failures, observations); + } + } finally { + warmedUp.countDown(); + } + + // The state of the latch is read before the pass, so the pass which observes the + // finished writers runs completely after the last write. Every reader therefore + // sees the final version of every chunk, which makes the counters below an + // assertion instead of a coincidence. + boolean last = false; + + while (!last) { + last = written.getCount() == 0; + + for (int chunk = 0; chunk < TORN_CHUNK_COUNT; chunk++) { + inspectTornRead(region.readRaw(chunk, 0), chunk, failures, observations); + } + } + return null; + })); + } + + for (int chunk = 0; chunk < TORN_CHUNK_COUNT; chunk++) { + int index = chunk; + futures.add(executor.submit(() -> { + awaitStart(start); + awaitStart(warmedUp); + + try { + // Every writer has to finish its first version before any of them starts the + // second one. A range which was freed by a second version is exactly as + // large as a first version needs, so without this barrier the allocator + // would hand a recycled range to a first version and the file would no + // longer end up at the size this test computes below. The recycled range + // itself is driven by the test which is named for it. + try { + region.writeRaw(index, 0, ChunkCompression.ZLIB, tornPayload(index, 1)); + } finally { + grown.countDown(); + } + awaitStart(grown); + region.writeRaw(index, 0, ChunkCompression.ZLIB, tornPayload(index, 2)); + } finally { + written.countDown(); + } + return null; + })); + } + start.countDown(); + awaitAll(futures); + + for (int chunk = 0; chunk < TORN_CHUNK_COUNT; chunk++) { + assertArrayEquals(tornPayload(chunk, TORN_SECTORS.length - 1), read(region, chunk, 0), "chunk " + chunk + " lost its last version"); + } + } + + assertTrue(failures.isEmpty(), "a reader observed a payload which is not a single version: " + failures); + assertTrue( + observations.get(0) >= TORN_READER_COUNT * TORN_CHUNK_COUNT, + "every reader has to observe the old version before the writers are released but only " + + observations.get(0) + " reads saw it" + ); + assertTrue( + observations.get(TORN_SECTORS.length - 1) >= TORN_READER_COUNT * TORN_CHUNK_COUNT, + "every reader has to observe the final version after the writers finished but only " + + observations.get(TORN_SECTORS.length - 1) + " reads saw it" + ); + assertEquals( + (long) TORN_TOTAL_SECTORS * RegionConstants.SECTOR_SIZE, Files.size(path), + "the barrier of this test rules every recycled sector out, so the file has to end at the computed size" + ); + assertSectorTableIsDisjoint(path); + } + + @Test + void testReadersNeverObserveASectorWhichWasRecycledWhileTheyRead() throws IOException, InterruptedException, ExecutionException { + for (int attempt = 0; attempt < RECYCLE_ATTEMPTS; attempt++) { + readWhileTheSectorsAreRecycled(regionPath(attempt)); + } + } + + /** + * Shrinks and grows a single chunk while other chunks compete for the range it frees. + *

+ * This is the scenario the torn read test deliberately keeps out of its way with its barrier. + * The observed chunk alternates between the largest payload a location entry can address and a + * single sector, so every shrink frees a large range. The filler chunks request exactly that + * many sectors, so the first fit strategy of the allocator hands the freed range to one of them + * while a reader may still be somewhere inside it. A reader must never see those foreign bytes, + * must never see a header field of a foreign chunk and must never lose the chunk. + *

+ * + * @param path the path of the region file to work on + * @throws IOException if the region file cannot be used + * @throws InterruptedException if the waiting thread is interrupted + * @throws ExecutionException if a worker failed + */ + private void readWhileTheSectorsAreRecycled(Path path) throws IOException, InterruptedException, ExecutionException { + byte[] large = marked(RECYCLE_LARGE_SECTORS * RegionConstants.SECTOR_SIZE - 5, RECYCLE_LARGE_MARKER); + byte[] small = marked(RECYCLE_SMALL_SECTORS * RegionConstants.SECTOR_SIZE - 5, RECYCLE_SMALL_MARKER); + Queue failures = new ConcurrentLinkedQueue<>(); + AtomicInteger reads = new AtomicInteger(); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch written = new CountDownLatch(1 + RECYCLE_FILLER_COUNT); + + try (RegionFile region = RegionFile.open(path); + ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + region.writeRaw(0, 0, ChunkCompression.ZLIB, large); + + List> futures = new ArrayList<>(1 + RECYCLE_FILLER_COUNT + RECYCLE_READER_COUNT); + + futures.add(executor.submit(() -> { + awaitStart(start); + + try { + for (int round = 0; round < RECYCLE_ROUNDS; round++) { + region.writeRaw(0, 0, ChunkCompression.ZLIB, small); + region.writeRaw(0, 0, ChunkCompression.ZLIB, large); + } + } finally { + written.countDown(); + } + return null; + })); + + for (int filler = 0; filler < RECYCLE_FILLER_COUNT; filler++) { + byte[] payload = marked(RECYCLE_LARGE_SECTORS * RegionConstants.SECTOR_SIZE - 5, (byte) (0x80 + filler)); + int chunk = filler + 1; + futures.add(executor.submit(() -> { + awaitStart(start); + + try { + for (int round = 0; round < RECYCLE_ROUNDS * 2; round++) { + region.writeRaw(chunk, 0, ChunkCompression.ZLIB, payload); + } + } finally { + written.countDown(); + } + return null; + })); + } + + for (int reader = 0; reader < RECYCLE_READER_COUNT; reader++) { + futures.add(executor.submit(() -> { + awaitStart(start); + + while (written.getCount() > 0) { + inspectRecycledRead(region, failures); + reads.incrementAndGet(); + } + return null; + })); + } + start.countDown(); + awaitAll(futures); + + assertArrayEquals(large, read(region, 0, 0), "the observed chunk lost its last version"); + } + + assertTrue(failures.isEmpty(), "a reader observed bytes of a recycled sector: " + failures); + assertTrue(reads.get() > 0, "no reader ever read the observed chunk"); + assertSectorTableIsDisjoint(path); + } + + @Test + void testConcurrentStorageSwitchesKeepTheExternalFileAndTheHeaderInSync() throws IOException, InterruptedException, ExecutionException { + for (int attempt = 0; attempt < ATTEMPTS; attempt++) { + switchStorageConcurrently(regionPath(attempt), attempt); + } + } + + /** + * Writes the same chunk from two threads where one payload needs an external file and the other + * one does not. + *

+ * The header entry decides where a reader looks for the payload, so the entry and the external + * file have to change together. A writer which creates the file before it owns the header, or + * removes it after it gave the header up, lets the other writer slip in between: the header then + * claims an external payload while the file behind it is already gone, which breaks the chunk + * for good instead of only for the moment. + *

+ * + * @param path the path of the region file to work on + * @param chunkZ the chunk z coordinate the attempt uses so its external file is its own + * @throws IOException if the region file cannot be used + * @throws InterruptedException if the waiting thread is interrupted + * @throws ExecutionException if a worker failed + */ + private void switchStorageConcurrently(Path path, int chunkZ) throws IOException, InterruptedException, ExecutionException { + byte[] external = marked(EXTERNAL_PAYLOAD_LENGTH, EXTERNAL_MARKER); + byte[] inline = marked(INLINE_PAYLOAD_LENGTH, INLINE_MARKER); + Queue failures = new ConcurrentLinkedQueue<>(); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch written = new CountDownLatch(1); + + try (RegionFile region = RegionFile.open(path); + ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + region.writeRaw(0, chunkZ, ChunkCompression.ZLIB, inline); + + List> futures = new ArrayList<>(2 + SWITCH_READER_COUNT); + + // The external payload is three orders of magnitude larger than the inline one, so a + // fixed round count would let the inline writer finish long before the first one even + // reached its second round. The inline writer therefore keeps going until the external + // one is done, which keeps both of them in the same window for the whole run. + futures.add(executor.submit(() -> { + awaitStart(start); + + try { + for (int round = 0; round < SWITCH_ROUNDS; round++) { + region.writeRaw(0, chunkZ, ChunkCompression.ZLIB, external); + } + } finally { + written.countDown(); + } + return null; + })); + + futures.add(executor.submit(() -> { + awaitStart(start); + + while (written.getCount() > 0) { + region.writeRaw(0, chunkZ, ChunkCompression.ZLIB, inline); + } + return null; + })); + + for (int reader = 0; reader < SWITCH_READER_COUNT; reader++) { + futures.add(executor.submit(() -> { + awaitStart(start); + + while (written.getCount() > 0) { + inspectSwitchedRead(region, chunkZ, failures); + } + return null; + })); + } + start.countDown(); + awaitAll(futures); + + inspectSwitchedRead(region, chunkZ, failures); + } + + assertTrue(failures.isEmpty(), "a reader could not follow the storage the header points at: " + failures); + + Path externalFile = this.tempDir.resolve("c.0." + chunkZ + ".mcc"); + assertEquals( + ChunkCompression.isExternal(storedScheme(path, 0, chunkZ)), Files.exists(externalFile), + "the header entry and the external file of the chunk describe a different storage" + ); + + try (RegionFile reopened = RegionFile.open(path)) { + byte[] payload = read(reopened, 0, chunkZ); + + assertTrue( + Arrays.equals(external, payload) || Arrays.equals(inline, payload), + "the chunk holds " + payload.length + " bytes which belong to no version" + ); + } + } + + @Test + void testConcurrentGrowAndShrinkCyclesKeepTheFileConsistent() throws IOException, InterruptedException, ExecutionException { + for (int attempt = 0; attempt < ATTEMPTS; attempt++) { + growAndShrinkConcurrently(regionPath(attempt), attempt); + } + } + + /** + * Rewrites every chunk of a region file with changing payload sizes and verifies the result. + * The chunk z coordinate differs per attempt so the external files of two attempts cannot + * collide inside the shared temporary directory. + * + * @param path the path of the region file to work on + * @param chunkZ the chunk z coordinate every chunk of the attempt uses + * @throws IOException if the region file cannot be used + * @throws InterruptedException if the waiting thread is interrupted + * @throws ExecutionException if a writer failed + */ + private void growAndShrinkConcurrently(Path path, int chunkZ) throws IOException, InterruptedException, ExecutionException { + // Every chunk is owned by exactly one thread, so the order of its own writes is defined + // while the writes of the different chunks overlap. The sizes cross both a sector boundary + // and the limit of an inline chunk, so the allocator has to free and reuse ranges of very + // different lengths while other threads allocate. A missing lock lets two of those ranges + // overlap, and a lost external file makes the payload unreadable. + int chunkCount = 4; + int rounds = 3; + CountDownLatch start = new CountDownLatch(1); + + try (RegionFile region = RegionFile.open(path); + ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(chunkCount); + + for (int index = 0; index < chunkCount; index++) { + int chunk = index; + futures.add(executor.submit(() -> { + awaitStart(start); + + for (int round = 0; round < rounds; round++) { + for (int size = 0; size < CYCLE_SIZES.length; size++) { + region.writeRaw(chunk, chunkZ, ChunkCompression.ZLIB, marked(CYCLE_SIZES[size], (byte) (chunk * 16 + round * 4 + size))); + } + } + region.writeRaw(chunk, chunkZ, ChunkCompression.ZLIB, marked(CYCLE_SIZES[chunk], (byte) (100 + chunk))); + return null; + })); + } + start.countDown(); + awaitAll(futures); + } + + assertSectorTableIsDisjoint(path); + + try (RegionFile reopened = RegionFile.open(path)) { + for (int chunk = 0; chunk < chunkCount; chunk++) { + byte[] expected = marked(CYCLE_SIZES[chunk], (byte) (100 + chunk)); + + assertArrayEquals(expected, read(reopened, chunk, chunkZ), "chunk " + chunk + " does not hold its last payload"); + assertEquals( + isExternal(CYCLE_SIZES[chunk]), Files.exists(this.tempDir.resolve("c." + chunk + "." + chunkZ + ".mcc")), + "the external file of the chunk " + chunk + " does not match its last payload" + ); + } + } + } + + @Test + void testClosingDuringConcurrentAccessFailsCleanlyAndKeepsTheFileReadable() throws IOException, InterruptedException, ExecutionException { + for (int attempt = 0; attempt < ATTEMPTS; attempt++) { + closeDuringConcurrentAccess(regionPath(attempt)); + } + } + + /** + * Closes a region file while readers and writers work on it and verifies the result. + * + * @param path the path of the region file to work on + * @throws IOException if the region file cannot be used + * @throws InterruptedException if the waiting thread is interrupted + * @throws ExecutionException if a worker failed for an unexpected reason + */ + private void closeDuringConcurrentAccess(Path path) throws IOException, InterruptedException, ExecutionException { + // Closing a file while other threads work on it must never leave a header entry which + // points at a range the file does not hold. Every worker has to report the shutdown as an + // IOException, because any other exception type would reach a caller that only expects an + // input output failure. The file is reopened afterwards, which rebuilds the sector usage + // from the header and therefore rejects a layout which was destroyed by the close. + int prefilled = 16; + int writerCount = 4; + int readerCount = 8; + List payloads = new ArrayList<>(prefilled + writerCount); + + for (int index = 0; index < prefilled + writerCount; index++) { + payloads.add(marked(RegionConstants.SECTOR_SIZE - 5, (byte) (index + 1))); + } + + CountDownLatch start = new CountDownLatch(1); + CountDownLatch working = new CountDownLatch(writerCount + readerCount); + AtomicInteger completedOperations = new AtomicInteger(); + List> failures = new ArrayList<>(writerCount + readerCount); + + try (RegionFile region = RegionFile.open(path); + ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + for (int index = 0; index < prefilled; index++) { + region.writeRaw(index, 0, ChunkCompression.ZLIB, payloads.get(index)); + } + + List> futures = new ArrayList<>(writerCount + readerCount); + + for (int index = 0; index < writerCount + readerCount; index++) { + boolean writer = index < writerCount; + int slot = writer ? prefilled + index : index - writerCount; + AtomicReference failure = new AtomicReference<>(); + failures.add(failure); + + futures.add(executor.submit(() -> { + awaitStart(start); + boolean reported = false; + + try { + while (true) { + if (writer) { + region.writeRaw(slot, 0, ChunkCompression.ZLIB, payloads.get(slot)); + } else { + assertNotNull(region.readRaw(slot % prefilled, 0)); + } + completedOperations.incrementAndGet(); + + if (!reported) { + reported = true; + working.countDown(); + } + } + } catch (Throwable throwable) { + failure.set(throwable); + } finally { + if (!reported) { + working.countDown(); + } + } + return null; + })); + } + start.countDown(); + awaitStart(working); + region.close(); + awaitAll(futures); + } + + assertTrue(completedOperations.get() >= writerCount + readerCount, "every worker has to run at least once before the close"); + + for (int index = 0; index < failures.size(); index++) { + Throwable failure = failures.get(index).get(); + + assertNotNull(failure, "the worker " + index + " never noticed the closed file"); + assertInstanceOf(IOException.class, failure, "the worker " + index + " reported " + failure); + } + + assertSectorTableIsDisjoint(path); + + try (RegionFile reopened = RegionFile.open(path)) { + for (int index = 0; index < prefilled; index++) { + assertArrayEquals(payloads.get(index), read(reopened, index, 0), "the close destroyed the chunk " + index); + } + for (int index = prefilled; index < prefilled + writerCount; index++) { + RegionFile.RawChunk chunk = reopened.readRaw(index, 0); + + if (chunk != null) { + assertArrayEquals(payloads.get(index), chunk.payload(), "the close truncated the chunk " + index); + } + } + } + } + + /** + * Builds a payload in which every byte carries the given marker. + * A payload of a single repeated byte turns any mix of two payloads into a mismatch, no matter + * at which offset the two were mixed. + * + * @param length the amount of bytes the payload holds + * @param marker the byte every position of the payload carries + * @return the created payload + */ + private static byte[] marked(int length, byte marker) { + byte[] payload = new byte[length]; + Arrays.fill(payload, marker); + return payload; + } + + /** + * Builds the payload of a version of a chunk of the torn read test. + * + * @param chunk the index of the chunk + * @param version the index of the version + * @return the created payload + */ + private static byte[] tornPayload(int chunk, int version) { + return marked(TORN_SECTORS[version] * RegionConstants.SECTOR_SIZE - 5, tornMarker(chunk, version)); + } + + /** + * Builds the marker byte of a version of a chunk of the torn read test. + * The marker holds both values so a payload which was written for another chunk is detected as + * well as a payload which mixes two versions. + * + * @param chunk the index of the chunk + * @param version the index of the version + * @return the marker byte of the version + */ + private static byte tornMarker(int chunk, int version) { + return (byte) ((chunk << 2) | version); + } + + /** + * Verifies that the given chunk holds exactly one version of exactly one chunk. + * The version is derived from the length of the payload, which differs per version, and every + * byte has to carry the marker of that version afterwards. + * + * @param chunk the chunk which was read + * @param chunkIndex the index of the chunk which was requested + * @param failures the queue which collects the description of every violation + * @param observations the counters which record how often a version was observed + */ + private static void inspectTornRead(RegionFile.RawChunk chunk, int chunkIndex, Queue failures, AtomicIntegerArray observations) { + if (chunk == null) { + failures.add("the chunk " + chunkIndex + " disappeared while it was rewritten"); + return; + } + + byte[] payload = chunk.payload(); + int version = -1; + + for (int candidate = 0; candidate < TORN_SECTORS.length; candidate++) { + if (payload.length == TORN_SECTORS[candidate] * RegionConstants.SECTOR_SIZE - 5) { + version = candidate; + break; + } + } + + if (version < 0) { + failures.add("the chunk " + chunkIndex + " reported " + payload.length + " bytes which belong to no version"); + return; + } + + byte expected = tornMarker(chunkIndex, version); + + for (int offset = 0; offset < payload.length; offset++) { + if (payload[offset] != expected) { + failures.add( + "the chunk " + chunkIndex + " holds " + payload[offset] + " at the offset " + offset + + " while its version " + version + " expects " + expected + ); + return; + } + } + observations.incrementAndGet(version); + } + + /** + * Reads the observed chunk of the recycling test once and records every deviation. + *

+ * The chunk only ever holds one of two payloads, so both the length and the marker of a read are + * known up front. A failure of the read itself is recorded as well, because a header field which + * was overwritten by a foreign chunk shows up as a rejected length or an unknown scheme. + *

+ * + * @param region the region file to read from + * @param failures the queue which collects the description of every violation + */ + private static void inspectRecycledRead(RegionFile region, Queue failures) { + RegionFile.RawChunk chunk; + + try { + chunk = region.readRaw(0, 0); + } catch (IOException exception) { + failures.add("the observed chunk could not be read: " + exception); + return; + } + + if (chunk == null) { + failures.add("the observed chunk disappeared while it was rewritten"); + return; + } + + byte[] payload = chunk.payload(); + byte expected; + + if (payload.length == RECYCLE_LARGE_SECTORS * RegionConstants.SECTOR_SIZE - 5) { + expected = RECYCLE_LARGE_MARKER; + } else if (payload.length == RECYCLE_SMALL_SECTORS * RegionConstants.SECTOR_SIZE - 5) { + expected = RECYCLE_SMALL_MARKER; + } else { + failures.add("the observed chunk reported " + payload.length + " bytes which belong to no version"); + return; + } + + for (int offset = 0; offset < payload.length; offset++) { + if (payload[offset] != expected) { + failures.add( + "the observed chunk holds " + payload[offset] + " at the offset " + offset + " of " + + payload.length + " while it expects " + expected + ); + return; + } + } + } + + /** + * Reads the chunk of the storage switch test once and records every deviation. + * + * @param region the region file to read from + * @param chunkZ the chunk z coordinate the attempt uses + * @param failures the queue which collects the description of every violation + */ + private static void inspectSwitchedRead(RegionFile region, int chunkZ, Queue failures) { + RegionFile.RawChunk chunk; + + try { + chunk = region.readRaw(0, chunkZ); + } catch (IOException exception) { + failures.add("the chunk could not be read: " + exception); + return; + } + + if (chunk == null) { + failures.add("the chunk disappeared while it was rewritten"); + return; + } + + byte[] payload = chunk.payload(); + byte expected; + + if (payload.length == EXTERNAL_PAYLOAD_LENGTH) { + expected = EXTERNAL_MARKER; + } else if (payload.length == INLINE_PAYLOAD_LENGTH) { + expected = INLINE_MARKER; + } else { + failures.add("the chunk reported " + payload.length + " bytes which belong to no version"); + return; + } + + for (int offset = 0; offset < payload.length; offset++) { + if (payload[offset] != expected) { + failures.add( + "the chunk holds " + payload[offset] + " at the offset " + offset + " of " + payload.length + + " while it expects " + expected + ); + return; + } + } + } + + /** + * Reads the compression scheme byte a chunk carries straight from the region file on disk. + * + * @param path the path of the region file to inspect + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @return the scheme byte of the chunk including the external flag + * @throws IOException if the region file cannot be read + */ + private static int storedScheme(Path path, int chunkX, int chunkZ) throws IOException { + byte[] bytes = Files.readAllBytes(path); + int location = ByteBuffer.wrap(bytes).getInt(RegionConstants.locationOffset(RegionConstants.index(chunkX, chunkZ))); + + assertTrue(location != 0, "the chunk " + chunkX + "/" + chunkZ + " has no entry in the location table"); + return bytes[(location >>> 8) * RegionConstants.SECTOR_SIZE + RegionConstants.LENGTH_FIELD_SIZE] & 0xFF; + } + + /** + * Checks whether a payload of the given length is stored outside of the region file. + * + * @param length the amount of bytes the payload holds + * @return true if the payload needs an external file, otherwise false + */ + private static boolean isExternal(int length) { + return RegionConstants.sectorsFor(RegionConstants.LENGTH_FIELD_SIZE + RegionConstants.COMPRESSION_FIELD_SIZE + length) + > RegionConstants.MAX_SECTORS_PER_CHUNK; + } + + /** + * Reads the payload of a chunk and fails when the chunk is absent. + * + * @param region the region file to read from + * @param chunkX the absolute chunk x coordinate + * @param chunkZ the absolute chunk z coordinate + * @return the payload of the chunk + * @throws IOException if the chunk cannot be read + */ + private static byte[] read(RegionFile region, int chunkX, int chunkZ) throws IOException { + RegionFile.RawChunk chunk = region.readRaw(chunkX, chunkZ); + + assertNotNull(chunk, "the chunk " + chunkX + "/" + chunkZ + " is missing"); + return chunk.payload(); + } + + /** + * Verifies that no two entries of the location table describe overlapping sectors. + *

+ * The check reads the header straight from disk instead of asking the region file, so it also + * covers the case in which the in memory tables and the stored ones drifted apart. + *

+ * + * @param path the path of the region file to inspect + * @throws IOException if the region file cannot be read + */ + private static void assertSectorTableIsDisjoint(Path path) throws IOException { + byte[] bytes = Files.readAllBytes(path); + + assertEquals(0, bytes.length % RegionConstants.SECTOR_SIZE, "the region file is not aligned to the sector size"); + + ByteBuffer header = ByteBuffer.wrap(bytes); + int totalSectors = bytes.length / RegionConstants.SECTOR_SIZE; + int[] owner = new int[totalSectors]; + Arrays.fill(owner, -1); + + for (int index = 0; index < RegionConstants.ENTRY_COUNT; index++) { + int location = header.getInt(RegionConstants.locationOffset(index)); + + if (location == 0) { + continue; + } + + int offset = location >>> 8; + int count = location & 0xFF; + + assertTrue(offset >= RegionConstants.HEADER_SECTORS, "the entry " + index + " points into the header at the sector " + offset); + assertTrue(count > 0, "the entry " + index + " spans no sector at all"); + assertTrue(offset + count <= totalSectors, "the entry " + index + " ends behind the file at the sector " + (offset + count)); + + for (int sector = offset; sector < offset + count; sector++) { + assertEquals(-1, owner[sector], "the sector " + sector + " is claimed by the entries " + owner[sector] + " and " + index); + owner[sector] = index; + } + } + } + + /** + * Waits for the given latch and fails when it is not released in time. + * + * @param latch the latch to wait for + */ + private static void awaitStart(CountDownLatch latch) { + try { + assertTrue(latch.await(AWAIT_SECONDS, TimeUnit.SECONDS), "a worker waited too long for its barrier"); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + fail("a worker was interrupted while it waited for its barrier"); + } + } + + /** + * Waits for every given task and propagates the failure of the first broken one. + * + * @param futures the tasks to wait for + * @throws InterruptedException if the waiting thread is interrupted + * @throws ExecutionException if a task failed + */ + private static void awaitAll(List> futures) throws InterruptedException, ExecutionException { + for (Future future : futures) { + future.get(); + } + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/anvil/RegionFileTest.java b/src/test/java/net/theevilreaper/aves/instance/anvil/RegionFileTest.java new file mode 100644 index 00000000..6e5546fd --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/anvil/RegionFileTest.java @@ -0,0 +1,253 @@ +package net.theevilreaper.aves.instance.anvil; + +import net.theevilreaper.aves.FileTestBase; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.random.RandomGenerator; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the region file container which stores the raw chunk payloads. + * The tests only work on bytes so no Minestom server is required. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class RegionFileTest extends FileTestBase { + + private static final byte[] PAYLOAD = "a compressed chunk payload".getBytes(StandardCharsets.UTF_8); + + /** + * Creates the path of a region file inside the temporary directory of the test. + * + * @return the path of the region file + */ + private Path regionPath() { + return this.tempDir.resolve("r.0.0.mca"); + } + + @Test + void testOpeningAMissingFileCreatesTheHeader() throws IOException { + try (RegionFile ignored = RegionFile.open(regionPath())) { + assertTrue(Files.exists(regionPath())); + } + assertEquals(RegionConstants.HEADER_SIZE, Files.size(regionPath())); + } + + @Test + void testReadReturnsNullForAnAbsentChunk() throws IOException { + try (RegionFile region = RegionFile.open(regionPath())) { + assertNull(region.readRaw(0, 0)); + assertFalse(region.hasChunk(0, 0)); + } + } + + @Test + void testWrittenChunkCanBeReadBack() throws IOException { + try (RegionFile region = RegionFile.open(regionPath())) { + region.writeRaw(3, 7, ChunkCompression.ZLIB, PAYLOAD); + + RegionFile.RawChunk chunk = region.readRaw(3, 7); + + assertNotNull(chunk); + assertEquals(ChunkCompression.ZLIB, chunk.compression()); + assertArrayEquals(PAYLOAD, chunk.payload()); + assertTrue(region.hasChunk(3, 7)); + } + } + + @Test + void testChunksSurviveAReopen() throws IOException { + try (RegionFile region = RegionFile.open(regionPath())) { + region.writeRaw(1, 1, ChunkCompression.ZLIB, PAYLOAD); + } + + try (RegionFile region = RegionFile.open(regionPath())) { + RegionFile.RawChunk chunk = region.readRaw(1, 1); + + assertNotNull(chunk); + assertArrayEquals(PAYLOAD, chunk.payload()); + } + } + + @Test + void testDifferentChunksDoNotOverwriteEachOther() throws IOException { + byte[] other = "a completely different payload".getBytes(StandardCharsets.UTF_8); + + try (RegionFile region = RegionFile.open(regionPath())) { + region.writeRaw(0, 0, ChunkCompression.ZLIB, PAYLOAD); + region.writeRaw(31, 31, ChunkCompression.GZIP, other); + + assertArrayEquals(PAYLOAD, assertPresent(region.readRaw(0, 0)).payload()); + assertArrayEquals(other, assertPresent(region.readRaw(31, 31)).payload()); + } + } + + @Test + void testRewritingAChunkWithALargerPayloadKeepsTheNeighbourIntact() throws IOException { + byte[] large = new byte[RegionConstants.SECTOR_SIZE * 3]; + RandomGenerator.getDefault().nextBytes(large); + byte[] neighbour = "the neighbour must stay readable".getBytes(StandardCharsets.UTF_8); + + try (RegionFile region = RegionFile.open(regionPath())) { + region.writeRaw(0, 0, ChunkCompression.ZLIB, PAYLOAD); + region.writeRaw(1, 0, ChunkCompression.ZLIB, neighbour); + region.writeRaw(0, 0, ChunkCompression.ZLIB, large); + + assertArrayEquals(large, assertPresent(region.readRaw(0, 0)).payload()); + assertArrayEquals(neighbour, assertPresent(region.readRaw(1, 0)).payload()); + } + } + + @Test + void testTheFileStaysAlignedToTheSectorSize() throws IOException { + try (RegionFile region = RegionFile.open(regionPath())) { + region.writeRaw(0, 0, ChunkCompression.ZLIB, PAYLOAD); + region.writeRaw(5, 5, ChunkCompression.ZLIB, new byte[RegionConstants.SECTOR_SIZE + 17]); + } + + assertEquals(0, Files.size(regionPath()) % RegionConstants.SECTOR_SIZE); + } + + @Test + void testTheLengthFieldFollowsTheSpecification() throws IOException { + try (RegionFile region = RegionFile.open(regionPath())) { + region.writeRaw(0, 0, ChunkCompression.ZLIB, PAYLOAD); + } + + // The specification defines the length field as the compression byte plus the payload. + // Minestom writes four bytes too many here, this loader must not repeat that. + assertEquals(PAYLOAD.length + 1, readChunkLengthField()); + } + + @Test + void testDeletingAChunkClearsItsEntry() throws IOException { + try (RegionFile region = RegionFile.open(regionPath())) { + region.writeRaw(2, 2, ChunkCompression.ZLIB, PAYLOAD); + region.delete(2, 2); + + assertFalse(region.hasChunk(2, 2)); + assertNull(region.readRaw(2, 2)); + } + } + + @Test + void testAnOversizedChunkIsStoredInAnExternalFile() throws IOException { + byte[] oversized = new byte[RegionConstants.MAX_SECTORS_PER_CHUNK * RegionConstants.SECTOR_SIZE + 1]; + RandomGenerator.getDefault().nextBytes(oversized); + + try (RegionFile region = RegionFile.open(regionPath())) { + region.writeRaw(4, 6, ChunkCompression.ZLIB, oversized); + + assertTrue(Files.exists(this.tempDir.resolve("c.4.6.mcc"))); + assertArrayEquals(oversized, assertPresent(region.readRaw(4, 6)).payload()); + } + } + + @Test + void testShrinkingAnExternalChunkRemovesTheExternalFile() throws IOException { + byte[] oversized = new byte[RegionConstants.MAX_SECTORS_PER_CHUNK * RegionConstants.SECTOR_SIZE + 1]; + RandomGenerator.getDefault().nextBytes(oversized); + + try (RegionFile region = RegionFile.open(regionPath())) { + region.writeRaw(4, 6, ChunkCompression.ZLIB, oversized); + region.writeRaw(4, 6, ChunkCompression.ZLIB, PAYLOAD); + + assertFalse(Files.exists(this.tempDir.resolve("c.4.6.mcc"))); + assertArrayEquals(PAYLOAD, assertPresent(region.readRaw(4, 6)).payload()); + } + } + + @Test + void testAFileWithATruncatedHeaderIsRejected() throws IOException { + Files.write(regionPath(), new byte[RegionConstants.SECTOR_SIZE]); + + assertThrows(IOException.class, () -> RegionFile.open(regionPath()).close()); + } + + @Test + void testAUsageAfterCloseIsRejected() throws IOException { + RegionFile region = RegionFile.open(regionPath()); + region.close(); + + assertThrows(IOException.class, () -> region.readRaw(0, 0)); + } + + @Test + void testConcurrentWritesNeverCorruptEachOther() throws IOException, InterruptedException, ExecutionException { + int chunkCount = 64; + List payloads = new ArrayList<>(chunkCount); + + for (int i = 0; i < chunkCount; i++) { + byte[] payload = new byte[512 + i * 64]; + RandomGenerator.getDefault().nextBytes(payload); + payloads.add(payload); + } + + try (RegionFile region = RegionFile.open(regionPath()); + ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(chunkCount); + + for (int i = 0; i < chunkCount; i++) { + int index = i; + futures.add(executor.submit(() -> { + region.writeRaw(index % 32, index / 32, ChunkCompression.ZLIB, payloads.get(index)); + return null; + })); + } + for (Future future : futures) { + future.get(); + } + + for (int i = 0; i < chunkCount; i++) { + assertArrayEquals(payloads.get(i), assertPresent(region.readRaw(i % 32, i / 32)).payload(), "chunk " + i + " was corrupted"); + } + } + } + + /** + * Reads the length field of the chunk which is stored in the first data sector. + * + * @return the value of the length field + * @throws IOException if the region file cannot be read + */ + private int readChunkLengthField() throws IOException { + try (FileChannel channel = FileChannel.open(regionPath(), StandardOpenOption.READ)) { + ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES); + channel.read(buffer, RegionConstants.HEADER_SIZE); + return buffer.flip().getInt(); + } + } + + /** + * Asserts that the given chunk is present and returns it. + * + * @param chunk the chunk to check + * @return the given chunk + */ + private RegionFile.RawChunk assertPresent(RegionFile.RawChunk chunk) { + assertNotNull(chunk); + return chunk; + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/anvil/SectionCodecTest.java b/src/test/java/net/theevilreaper/aves/instance/anvil/SectionCodecTest.java new file mode 100644 index 00000000..820188aa --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/anvil/SectionCodecTest.java @@ -0,0 +1,240 @@ +package net.theevilreaper.aves.instance.anvil; + +import net.kyori.adventure.nbt.BinaryTagTypes; +import net.kyori.adventure.nbt.CompoundBinaryTag; +import net.kyori.adventure.nbt.ListBinaryTag; +import net.kyori.adventure.nbt.LongArrayBinaryTag; +import net.kyori.adventure.nbt.StringBinaryTag; +import org.jetbrains.annotations.Nullable; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the conversion between the palette container of a section and the palette + * representation of the codec. A fake resolver keeps the tests free of a Minestom server. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class SectionCodecTest { + + private static final int BLOCK_ENTRIES = 4096; + private static final int BLOCK_MIN_BITS = 4; + + /** + * A resolver which maps a fixed set of names to ids without touching any registry. + */ + private static final class FakeResolver implements PaletteEntryResolver { + + private final List known = new ArrayList<>(List.of("minecraft:air", "minecraft:stone", "minecraft:dirt")); + private final List unresolved = new ArrayList<>(); + + @Override + public int toId(String name, @Nullable CompoundBinaryTag properties) { + int index = this.known.indexOf(name); + + if (index < 0) { + this.unresolved.add(name); + return 0; + } + return properties == null ? index : index + 100; + } + + @Override + public CompoundBinaryTag toEntry(int id) { + return CompoundBinaryTag.builder().putString("Name", this.known.get(id % 100)).build(); + } + } + + /** + * Builds a palette container in the shape the Anvil format uses. + * + * @param names the names of the palette entries + * @param data the packed indices or null if the container holds a single value + * @return the created container + */ + private static CompoundBinaryTag container(List names, long @Nullable [] data) { + ListBinaryTag.Builder palette = ListBinaryTag.builder(BinaryTagTypes.COMPOUND); + + for (String name : names) { + palette.add(CompoundBinaryTag.builder().put("Name", StringBinaryTag.stringBinaryTag(name)).build()); + } + + CompoundBinaryTag.Builder builder = CompoundBinaryTag.builder().put("palette", palette.build()); + + if (data != null) { + builder.put("data", LongArrayBinaryTag.longArrayBinaryTag(data)); + } + return builder.build(); + } + + @Test + void testDecodingASingleEntryContainerYieldsASingleValue() throws IOException { + CompoundBinaryTag container = container(List.of("minecraft:stone"), null); + + PaletteData data = SectionCodec.decode(container, new FakeResolver(), BLOCK_ENTRIES, BLOCK_MIN_BITS); + + assertTrue(data.isSingleValue()); + assertEquals(1, data.singleValue()); + } + + @Test + void testDecodingResolvesEveryPaletteEntry() throws IOException { + int[] indices = new int[BLOCK_ENTRIES]; + indices[0] = 1; + indices[1] = 2; + CompoundBinaryTag container = container( + List.of("minecraft:air", "minecraft:stone", "minecraft:dirt"), + BitPacker.pack(indices, BLOCK_MIN_BITS) + ); + + int[] values = SectionCodec.decode(container, new FakeResolver(), BLOCK_ENTRIES, BLOCK_MIN_BITS).unpack(); + + assertEquals(1, values[0]); + assertEquals(2, values[1]); + assertEquals(0, values[2]); + } + + @Test + void testDecodingPassesThePropertiesToTheResolver() throws IOException { + CompoundBinaryTag entry = CompoundBinaryTag.builder() + .putString("Name", "minecraft:stone") + .put("Properties", CompoundBinaryTag.builder().putString("axis", "y").build()) + .build(); + CompoundBinaryTag container = CompoundBinaryTag.builder() + .put("palette", ListBinaryTag.builder(BinaryTagTypes.COMPOUND).add(entry).build()) + .build(); + + PaletteData data = SectionCodec.decode(container, new FakeResolver(), BLOCK_ENTRIES, BLOCK_MIN_BITS); + + assertEquals(101, data.singleValue()); + } + + @Test + void testDecodingFailsForAMissingPalette() { + CompoundBinaryTag container = CompoundBinaryTag.empty(); + + assertThrows(IOException.class, () -> SectionCodec.decode(container, new FakeResolver(), BLOCK_ENTRIES, BLOCK_MIN_BITS)); + } + + @Test + void testDecodingFailsForAPaletteEntryWithoutAName() { + CompoundBinaryTag container = CompoundBinaryTag.builder() + .put("palette", ListBinaryTag.builder(BinaryTagTypes.COMPOUND).add(CompoundBinaryTag.empty()).build()) + .build(); + + assertThrows(IOException.class, () -> SectionCodec.decode(container, new FakeResolver(), BLOCK_ENTRIES, BLOCK_MIN_BITS)); + } + + @Test + void testAnUnknownNameFallsBackInsteadOfFailing() throws IOException { + FakeResolver resolver = new FakeResolver(); + CompoundBinaryTag container = container(List.of("minecraft:mystery"), null); + + PaletteData data = SectionCodec.decode(container, resolver, BLOCK_ENTRIES, BLOCK_MIN_BITS); + + assertEquals(0, data.singleValue()); + assertEquals(List.of("minecraft:mystery"), resolver.unresolved); + } + + @Test + void testEncodingASingleValueOmitsTheDataArray() { + CompoundBinaryTag container = SectionCodec.encode(PaletteData.single(1, BLOCK_ENTRIES), new FakeResolver()); + + assertEquals(1, container.getList("palette").size()); + assertEquals(null, container.get("data")); + } + + @Test + void testEncodingWritesThePaletteAndTheData() { + int[] values = new int[BLOCK_ENTRIES]; + values[0] = 1; + values[1] = 2; + + CompoundBinaryTag container = SectionCodec.encode(PaletteData.encode(values, BLOCK_MIN_BITS), new FakeResolver()); + + assertEquals(3, container.getList("palette").size()); + assertTrue(container.get("data") instanceof LongArrayBinaryTag); + } + + @Test + void testDecodingBiomesReadsAPaletteOfPlainStrings() throws IOException { + // Unlike blocks, the format stores the biome palette as a list of names without properties. + ListBinaryTag palette = ListBinaryTag.builder(BinaryTagTypes.STRING) + .add(StringBinaryTag.stringBinaryTag("minecraft:air")) + .add(StringBinaryTag.stringBinaryTag("minecraft:dirt")) + .build(); + int[] indices = new int[64]; + indices[3] = 1; + CompoundBinaryTag container = CompoundBinaryTag.builder() + .put("palette", palette) + .put("data", LongArrayBinaryTag.longArrayBinaryTag(BitPacker.pack(indices, 1))) + .build(); + + int[] values = SectionCodec.decodeBiomes(container, new FakeResolver(), 64, 1).unpack(); + + assertEquals(2, values[3]); + assertEquals(0, values[0]); + } + + @Test + void testDecodingASingleBiomeNeedsNoData() throws IOException { + CompoundBinaryTag container = CompoundBinaryTag.builder() + .put("palette", ListBinaryTag.builder(BinaryTagTypes.STRING) + .add(StringBinaryTag.stringBinaryTag("minecraft:dirt")) + .build()) + .build(); + + PaletteData data = SectionCodec.decodeBiomes(container, new FakeResolver(), 64, 1); + + assertTrue(data.isSingleValue()); + assertEquals(2, data.singleValue()); + } + + @Test + void testEncodingBiomesWritesPlainStrings() { + CompoundBinaryTag container = SectionCodec.encodeBiomes(PaletteData.single(1, 64), new FakeResolver()); + + assertEquals(BinaryTagTypes.STRING, container.getList("palette").elementType()); + assertEquals("minecraft:stone", container.getList("palette").getString(0)); + } + + @Test + void testBiomesSurviveARoundTrip() throws IOException { + FakeResolver resolver = new FakeResolver(); + int[] values = new int[64]; + + for (int i = 0; i < values.length; i++) { + values[i] = i % 3; + } + + CompoundBinaryTag encoded = SectionCodec.encodeBiomes(PaletteData.encode(values, 1), resolver); + PaletteData decoded = SectionCodec.decodeBiomes(encoded, resolver, 64, 1); + + assertArrayEquals(values, decoded.unpack()); + } + + @Test + void testAContainerSurvivesARoundTrip() throws IOException { + FakeResolver resolver = new FakeResolver(); + int[] values = new int[BLOCK_ENTRIES]; + + for (int i = 0; i < values.length; i++) { + values[i] = i % 3; + } + + CompoundBinaryTag encoded = SectionCodec.encode(PaletteData.encode(values, BLOCK_MIN_BITS), resolver); + PaletteData decoded = SectionCodec.decode(encoded, resolver, BLOCK_ENTRIES, BLOCK_MIN_BITS); + + assertArrayEquals(values, decoded.unpack()); + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/anvil/SectorAllocatorTest.java b/src/test/java/net/theevilreaper/aves/instance/anvil/SectorAllocatorTest.java new file mode 100644 index 00000000..e0e527c3 --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/anvil/SectorAllocatorTest.java @@ -0,0 +1,109 @@ +package net.theevilreaper.aves.instance.anvil; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the sector allocation logic which is the core of the region file space management. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class SectorAllocatorTest { + + @Test + void testFreshAllocatorReservesTheHeaderSectors() { + SectorAllocator allocator = new SectorAllocator(RegionConstants.HEADER_SECTORS); + + assertEquals(RegionConstants.HEADER_SECTORS, allocator.totalSectors()); + assertFalse(allocator.isFree(0)); + assertFalse(allocator.isFree(1)); + } + + @Test + void testAllocateGrowsBeyondTheHeaderWhenNoFreeSpaceExists() { + SectorAllocator allocator = new SectorAllocator(RegionConstants.HEADER_SECTORS); + + assertEquals(2, allocator.allocate(3)); + assertEquals(5, allocator.totalSectors()); + } + + @Test + void testAllocateReusesFreedSectorsWithFirstFit() { + SectorAllocator allocator = new SectorAllocator(RegionConstants.HEADER_SECTORS); + int first = allocator.allocate(2); + int second = allocator.allocate(2); + allocator.free(first, 2); + + assertEquals(first, allocator.allocate(2)); + assertEquals(second + 2, allocator.totalSectors()); + } + + @Test + void testAllocateSkipsAGapThatIsTooSmall() { + SectorAllocator allocator = new SectorAllocator(RegionConstants.HEADER_SECTORS); + int first = allocator.allocate(1); + allocator.allocate(1); + allocator.free(first, 1); + + assertEquals(4, allocator.allocate(2)); + } + + @Test + void testFreedSectorsAreMarkedAsFree() { + SectorAllocator allocator = new SectorAllocator(RegionConstants.HEADER_SECTORS); + int offset = allocator.allocate(2); + allocator.free(offset, 2); + + assertTrue(allocator.isFree(offset)); + assertTrue(allocator.isFree(offset + 1)); + } + + @Test + void testReserveMarksAnExistingRangeAsUsed() { + SectorAllocator allocator = new SectorAllocator(10); + allocator.reserve(4, 2); + + assertFalse(allocator.isFree(4)); + assertFalse(allocator.isFree(5)); + assertTrue(allocator.isFree(6)); + } + + @Test + void testReserveRejectsAnOverlappingRange() { + SectorAllocator allocator = new SectorAllocator(10); + allocator.reserve(4, 2); + + assertThrows(IllegalStateException.class, () -> allocator.reserve(5, 2)); + } + + @ParameterizedTest + @ValueSource(ints = {0, -1}) + void testAllocateRejectsANonPositiveCount(int count) { + SectorAllocator allocator = new SectorAllocator(RegionConstants.HEADER_SECTORS); + + assertThrows(IllegalArgumentException.class, () -> allocator.allocate(count)); + } + + @Test + void testRepeatedAllocationsNeverOverlap() { + SectorAllocator allocator = new SectorAllocator(RegionConstants.HEADER_SECTORS); + boolean[] occupied = new boolean[512]; + + for (int i = 1; i <= 32; i++) { + int count = (i % 4) + 1; + int offset = allocator.allocate(count); + for (int sector = offset; sector < offset + count; sector++) { + assertFalse(occupied[sector], "sector " + sector + " was handed out twice"); + occupied[sector] = true; + } + } + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/light/ChunkBorderLightTest.java b/src/test/java/net/theevilreaper/aves/instance/light/ChunkBorderLightTest.java new file mode 100644 index 00000000..9a44f09c --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/light/ChunkBorderLightTest.java @@ -0,0 +1,236 @@ +package net.theevilreaper.aves.instance.light; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the exchange of light across a chunk border. Without it a torch placed next to the edge of + * a chunk lights its own chunk and leaves a hard dark line at the border. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class ChunkBorderLightTest { + + private static final int AIR = 0; + private static final int STONE = 1; + private static final int LAMP = 2; + + private static final BlockLightSource SOURCE = new BlockLightSource() { + + @Override + public int emission(int stateId) { + return stateId == LAMP ? 15 : 0; + } + + @Override + public boolean blocksFace(int stateId, BlockFace face) { + return stateId == STONE; + } + }; + + /** + * Builds the state ids of a chunk made of the given amount of air sections. + * + * @param sectionCount the amount of sections the chunk holds + * @return the state ids of every section + */ + private static List airChunk(int sectionCount) { + List sections = new ArrayList<>(sectionCount); + + for (int i = 0; i < sectionCount; i++) { + sections.add(new int[LightNibbles.BLOCK_COUNT]); + } + return sections; + } + + /** + * Converts the state ids of a chunk into opacity tables. + * + * @param sections the state ids of every section + * @return the opacity table of every section + */ + private static List tables(List sections) { + return sections.stream().map(states -> SectionOpacity.of(states, SOURCE)).toList(); + } + + /** + * Calculates the index of a block inside a section. + * + * @param x the x coordinate inside the section + * @param y the y coordinate inside the section + * @param z the z coordinate inside the section + * @return the index of the block + */ + private static int index(int x, int y, int z) { + return (y << 8) | (z << 4) | x; + } + + @Test + void testTheBorderOfAChunkCanBeRead() { + List sections = airChunk(1); + sections.get(0)[index(15, 8, 4)] = LAMP; + ChunkLightState state = ChunkLightState.blockLight(tables(sections)); + + byte[] border = state.border(BlockFace.EAST); + + assertEquals(LightNibbles.DIMENSION * LightNibbles.DIMENSION, border.length); + assertEquals(15, border[8 * LightNibbles.DIMENSION + 4], "the lamp sits on the east border"); + } + + @Test + void testLightEntersFromTheNeighbourAcrossTheBorder() { + // The source sits at the east edge of the western chunk. + List west = airChunk(1); + west.get(0)[index(15, 8, 8)] = LAMP; + ChunkLightState westState = ChunkLightState.blockLight(tables(west)); + + // The eastern chunk is empty and receives the light through its west border. + List east = airChunk(1); + List eastTables = tables(east); + ChunkLightState eastState = ChunkLightState.blockLight(eastTables); + + eastState.injectBorder(eastTables, BlockFace.WEST, westState.border(BlockFace.EAST)); + + assertEquals(14, eastState.get(0, 8, 8), "the first block behind the border loses one level"); + assertEquals(13, eastState.get(1, 8, 8)); + assertEquals(12, eastState.get(2, 8, 8)); + } + + @Test + void testLightEntersFromEveryHorizontalDirection() { + List neighbour = airChunk(1); + neighbour.get(0)[index(0, 8, 8)] = LAMP; + ChunkLightState neighbourState = ChunkLightState.blockLight(tables(neighbour)); + + List own = airChunk(1); + List ownTables = tables(own); + ChunkLightState ownState = ChunkLightState.blockLight(ownTables); + + ownState.injectBorder(ownTables, BlockFace.EAST, neighbourState.border(BlockFace.WEST)); + + assertEquals(14, ownState.get(15, 8, 8), "the light enters through the east border"); + } + + @Test + void testAWallAtTheBorderKeepsTheLightOut() { + List west = airChunk(1); + west.get(0)[index(15, 8, 8)] = LAMP; + ChunkLightState westState = ChunkLightState.blockLight(tables(west)); + + List east = airChunk(1); + + for (int y = 0; y < LightNibbles.DIMENSION; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + east.get(0)[index(0, y, z)] = STONE; + } + } + List eastTables = tables(east); + ChunkLightState eastState = ChunkLightState.blockLight(eastTables); + + eastState.injectBorder(eastTables, BlockFace.WEST, westState.border(BlockFace.EAST)); + + assertEquals(0, eastState.get(0, 8, 8), "the wall blocks the incoming light"); + assertEquals(0, eastState.get(1, 8, 8)); + } + + @Test + void testADarkNeighbourChangesNothing() { + List east = airChunk(1); + east.get(0)[index(8, 8, 8)] = LAMP; + List eastTables = tables(east); + ChunkLightState eastState = ChunkLightState.blockLight(eastTables); + + int beforeAtSource = eastState.get(8, 8, 8); + int beforeAtBorder = eastState.get(0, 8, 8); + + eastState.injectBorder(eastTables, BlockFace.WEST, new byte[LightNibbles.BLOCK_COUNT / LightNibbles.DIMENSION]); + + // The chunk lights itself, so the border block is not dark. What matters is that a dark + // neighbour neither adds nor removes anything. + assertEquals(beforeAtSource, eastState.get(8, 8, 8), "the own source must stay untouched"); + assertEquals(beforeAtBorder, eastState.get(0, 8, 8), "a dark neighbour must not change the border"); + } + + @Test + void testAVerticalFaceIsRejected() { + List sections = airChunk(1); + List tables = tables(sections); + ChunkLightState state = ChunkLightState.blockLight(tables); + + assertThrows(IllegalArgumentException.class, () -> state.border(BlockFace.TOP)); + } + + @Test + void testABorderOfTheWrongSizeIsRejected() { + List sections = airChunk(1); + List tables = tables(sections); + ChunkLightState state = ChunkLightState.blockLight(tables); + + assertThrows(IllegalArgumentException.class, () -> state.injectBorder(tables, BlockFace.WEST, new byte[3])); + } + + @Test + void testTheInjectedLightMatchesOneLargeCalculation() { + // Two chunks side by side with one source near the shared border. Injecting the border has + // to give the eastern chunk the same levels a single calculation over both would. + List west = airChunk(1); + west.get(0)[index(15, 8, 8)] = LAMP; + ChunkLightState westState = ChunkLightState.blockLight(tables(west)); + + List east = airChunk(1); + List eastTables = tables(east); + ChunkLightState eastState = ChunkLightState.blockLight(eastTables); + eastState.injectBorder(eastTables, BlockFace.WEST, westState.border(BlockFace.EAST)); + + // Expected levels: distance from the source, which sits one block west of x = 0. + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + int expected = Math.max(0, 14 - x); + assertEquals(expected, eastState.get(x, 8, 8), "mismatch at x " + x); + } + } + + @Test + void testAnInjectionReportsWhetherItRaisedALevel() { + List west = airChunk(1); + west.get(0)[index(15, 8, 8)] = LAMP; + ChunkLightState westState = ChunkLightState.blockLight(tables(west)); + + List east = airChunk(1); + List eastTables = tables(east); + ChunkLightState eastState = ChunkLightState.blockLight(eastTables); + + assertTrue(eastState.injectBorder(eastTables, BlockFace.WEST, westState.border(BlockFace.EAST)), + "the first injection raises the levels behind the border"); + assertFalse(eastState.injectBorder(eastTables, BlockFace.WEST, westState.border(BlockFace.EAST)), + "repeating the very same injection changes nothing"); + } + + @Test + void testTheBorderIsExchangedAcrossSectionsAsWell() { + List west = airChunk(2); + west.get(1)[index(15, 4, 8)] = LAMP; + ChunkLightState westState = ChunkLightState.blockLight(tables(west)); + + List east = airChunk(2); + List eastTables = tables(east); + ChunkLightState eastState = ChunkLightState.blockLight(eastTables); + + byte[] border = westState.border(BlockFace.EAST); + + assertEquals(2 * LightNibbles.DIMENSION * LightNibbles.DIMENSION, border.length, + "the border spans the full height of the column"); + + eastState.injectBorder(eastTables, BlockFace.WEST, border); + + assertTrue(eastState.get(0, 20, 8) > 0, "the light of the upper section has to cross too"); + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/light/ChunkLightPropagatorTest.java b/src/test/java/net/theevilreaper/aves/instance/light/ChunkLightPropagatorTest.java new file mode 100644 index 00000000..d9f088f9 --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/light/ChunkLightPropagatorTest.java @@ -0,0 +1,206 @@ +package net.theevilreaper.aves.instance.light; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the propagation across the sections of a chunk. Light that stops at a section border is + * the reason a per section propagation alone cannot be used for a real world. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class ChunkLightPropagatorTest { + + private static final int AIR = 0; + private static final int STONE = 1; + private static final int LAMP = 2; + + private static final BlockLightSource SOURCE = new BlockLightSource() { + + @Override + public int emission(int stateId) { + return stateId == LAMP ? 15 : 0; + } + + @Override + public boolean blocksFace(int stateId, BlockFace face) { + return stateId == STONE; + } + }; + + /** + * Builds the opacity tables of a chunk made of the given amount of air sections. + * + * @param sectionCount the amount of sections the chunk holds + * @return the state ids of every section + */ + private static List airChunk(int sectionCount) { + List sections = new ArrayList<>(sectionCount); + + for (int i = 0; i < sectionCount; i++) { + sections.add(new int[LightNibbles.BLOCK_COUNT]); + } + return sections; + } + + /** + * Converts the state ids of a chunk into opacity tables. + * + * @param sections the state ids of every section + * @return the opacity table of every section + */ + private static List tables(List sections) { + return sections.stream().map(states -> SectionOpacity.of(states, SOURCE)).toList(); + } + + /** + * Calculates the index of a block inside a section. + * + * @param x the x coordinate inside the section + * @param y the y coordinate inside the section + * @param z the z coordinate inside the section + * @return the index of the block + */ + private static int index(int x, int y, int z) { + return (y << 8) | (z << 4) | x; + } + + @Test + void testAChunkWithoutAnySourceStaysDark() { + List light = new ChunkLightPropagator().propagate(tables(airChunk(3))); + + assertEquals(3, light.size()); + for (LightNibbles section : light) { + assertTrue(section.isUniform()); + assertEquals(0, section.get(0, 0, 0)); + } + } + + @Test + void testLightCrossesTheBorderIntoTheSectionAbove() { + List sections = airChunk(2); + // A lamp at the very top of the lower section. + sections.get(0)[index(8, 15, 8)] = LAMP; + + List light = new ChunkLightPropagator().propagate(tables(sections)); + + assertEquals(15, light.get(0).get(8, 15, 8)); + assertEquals(14, light.get(1).get(8, 0, 8), "the section above has to receive the light"); + assertEquals(13, light.get(1).get(8, 1, 8)); + } + + @Test + void testLightCrossesTheBorderIntoTheSectionBelow() { + List sections = airChunk(2); + // A lamp at the very bottom of the upper section. + sections.get(1)[index(8, 0, 8)] = LAMP; + + List light = new ChunkLightPropagator().propagate(tables(sections)); + + assertEquals(15, light.get(1).get(8, 0, 8)); + assertEquals(14, light.get(0).get(8, 15, 8), "the section below has to receive the light"); + assertEquals(13, light.get(0).get(8, 14, 8)); + } + + @Test + void testLightReachesThroughSeveralSections() { + List sections = airChunk(3); + sections.get(0)[index(8, 14, 8)] = LAMP; + + List light = new ChunkLightPropagator().propagate(tables(sections)); + + // 15 at the source, one level lost per block: the second section starts at 14. + assertEquals(14, light.get(0).get(8, 15, 8)); + assertEquals(13, light.get(1).get(8, 0, 8)); + assertEquals(0, light.get(2).get(8, 0, 8), "the third section is out of range"); + } + + @Test + void testAnOpaqueLayerStopsTheLightAtTheBorder() { + List sections = airChunk(2); + sections.get(0)[index(8, 14, 8)] = LAMP; + + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + sections.get(0)[index(x, 15, z)] = STONE; + } + } + + List light = new ChunkLightPropagator().propagate(tables(sections)); + + assertEquals(0, light.get(1).get(8, 0, 8), "the closed layer must stop the light"); + } + + @Test + void testAnOpaqueLayerOnTheOtherSideOfTheBorderAlsoStops() { + List sections = airChunk(2); + sections.get(0)[index(8, 15, 8)] = LAMP; + + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + sections.get(1)[index(x, 0, z)] = STONE; + } + } + + List light = new ChunkLightPropagator().propagate(tables(sections)); + + assertEquals(0, light.get(1).get(8, 0, 8)); + assertEquals(0, light.get(1).get(8, 1, 8)); + } + + @Test + void testASourceInEverySectionIsHandled() { + List sections = airChunk(3); + sections.get(0)[index(1, 1, 1)] = LAMP; + sections.get(2)[index(14, 14, 14)] = LAMP; + + List light = new ChunkLightPropagator().propagate(tables(sections)); + + assertEquals(15, light.get(0).get(1, 1, 1)); + assertEquals(15, light.get(2).get(14, 14, 14)); + } + + @Test + void testTheResultIsIndependentOfTheSectionOrderOfSources() { + List ascending = airChunk(2); + ascending.get(0)[index(8, 15, 8)] = LAMP; + + List descending = airChunk(2); + descending.get(0)[index(8, 15, 8)] = LAMP; + + List first = new ChunkLightPropagator().propagate(tables(ascending)); + List second = new ChunkLightPropagator().propagate(tables(descending)); + + for (int section = 0; section < first.size(); section++) { + for (int y = 0; y < LightNibbles.DIMENSION; y++) { + assertEquals(first.get(section).get(8, y, 8), second.get(section).get(8, y, 8)); + } + } + } + + @Test + void testAnEmptyChunkIsRejected() { + assertThrows(IllegalArgumentException.class, () -> new ChunkLightPropagator().propagate(List.of())); + } + + @Test + void testThePropagatorCanBeReused() { + ChunkLightPropagator propagator = new ChunkLightPropagator(); + List lit = airChunk(2); + lit.get(0)[index(8, 15, 8)] = LAMP; + + List first = propagator.propagate(tables(lit)); + List second = propagator.propagate(tables(airChunk(2))); + + assertEquals(14, first.get(1).get(8, 0, 8), "the first result must stay untouched"); + assertEquals(0, second.get(1).get(8, 0, 8), "the reused buffers must not carry the previous run"); + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/light/ChunkLightServiceConcurrencyTest.java b/src/test/java/net/theevilreaper/aves/instance/light/ChunkLightServiceConcurrencyTest.java new file mode 100644 index 00000000..8c80f1bd --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/light/ChunkLightServiceConcurrencyTest.java @@ -0,0 +1,317 @@ +package net.theevilreaper.aves.instance.light; + +import net.minestom.server.instance.Chunk; +import net.minestom.server.instance.Instance; +import net.minestom.server.instance.Section; +import net.minestom.server.instance.block.Block; +import net.minestom.testing.Env; +import net.minestom.testing.extension.MicrotusExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Pins down that a single {@link ChunkLightService} may serve many threads at once. + *

+ * A server lights the chunks around a player in parallel and naturally keeps one service for the + * whole instance, so the shape of that usage decides whether the type is usable at all. The danger + * is that a broken service fails quietly: the light is written through + * {@link net.minestom.server.instance.light.Light#set(byte[])}, which clears the update flag of the + * section, so a wrong result is never recomputed by the server and only ever shows up as a dark + * patch in a world that nobody can explain. + *

+ *

+ * The tests therefore compare against a reference which was calculated one chunk after the other. + * Every worker owns its own chunk, so nothing the workers do can legitimately interfere; any + * difference from the reference can only come from state the service kept between two calls. The + * fixtures put their sources at a different height in every chunk so that two workers which drifted + * into each other produce visibly different columns instead of accidentally matching results. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ExtendWith(MicrotusExtension.class) +class ChunkLightServiceConcurrencyTest { + + /** + * The time a latch is waited for before the test is considered stuck. + */ + private static final long AWAIT_SECONDS = 60L; + + /** + * The amount of workers which use the service at the same time. + */ + private static final int THREAD_COUNT = 8; + + /** + * The amount of times every worker recalculates its chunk. + */ + private static final int ROUNDS = 12; + + @Test + void testOneServiceCalculatesTheSameBlockLightFromManyThreads(Env env) throws InterruptedException, ExecutionException { + Instance instance = env.createEmptyInstance(); + List chunks = lampChunks(instance); + List> expected = new ArrayList<>(chunks.size()); + + // The reference is taken one chunk after the other. A service which keeps nothing between + // two calls has to reproduce it no matter how many threads call it. + for (Chunk chunk : chunks) { + ChunkLightService reference = new ChunkLightService(); + reference.calculate(chunk); + expected.add(blockLightOf(chunk)); + } + + ChunkLightService shared = new ChunkLightService(); + + run(chunks, worker -> { + Chunk chunk = chunks.get(worker); + + for (int round = 0; round < ROUNDS; round++) { + shared.calculate(chunk); + assertLight(expected.get(worker), blockLightOf(chunk), "block light", worker, round); + } + }); + } + + @Test + void testOneServiceCalculatesTheSameSkyLightFromManyThreads(Env env) throws InterruptedException, ExecutionException { + Instance instance = env.createEmptyInstance(); + List chunks = ceilingChunks(instance); + List> expected = new ArrayList<>(chunks.size()); + + for (Chunk chunk : chunks) { + ChunkLightService reference = new ChunkLightService(); + reference.calculateSky(chunk); + expected.add(skyLightOf(chunk)); + } + + ChunkLightService shared = new ChunkLightService(); + + run(chunks, worker -> { + Chunk chunk = chunks.get(worker); + + for (int round = 0; round < ROUNDS; round++) { + shared.calculateSky(chunk); + assertLight(expected.get(worker), skyLightOf(chunk), "sky light", worker, round); + } + }); + } + + /** + * The work a single worker performs on the chunk it owns. + */ + @FunctionalInterface + private interface Worker { + + /** + * Runs the work of the worker with the given number. + * + * @param worker the number of the worker, which is also the index of its chunk + */ + void run(int worker); + } + + /** + * Starts one worker per chunk and waits for all of them. + *

+ * Every worker waits at the same barrier before it starts, so the calls overlap instead of + * running one after the other, which is the only way a shared buffer can be observed at all. + *

+ * + * @param chunks the chunks the workers operate on + * @param worker the work a single worker performs + * @throws InterruptedException if the waiting thread is interrupted + * @throws ExecutionException if a worker failed + */ + private static void run(List chunks, Worker worker) throws InterruptedException, ExecutionException { + CountDownLatch start = new CountDownLatch(1); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(chunks.size()); + + for (int index = 0; index < chunks.size(); index++) { + int number = index; + futures.add(executor.submit(() -> { + awaitStart(start); + worker.run(number); + return null; + })); + } + start.countDown(); + + for (Future future : futures) { + future.get(); + } + } + } + + /** + * Loads one chunk per worker and puts light sources and a wall into every one of them. + *

+ * The sources sit at a different height in every chunk, so the calculated columns differ from + * each other, and the wall forces the search around it instead of letting it finish in a + * straight sphere. + *

+ * + * @param instance the instance which holds the chunks + * @return the prepared chunks, one per worker + */ + private static List lampChunks(Instance instance) { + List chunks = new ArrayList<>(THREAD_COUNT); + + for (int index = 0; index < THREAD_COUNT; index++) { + Chunk chunk = instance.loadChunk(index, 0).join(); + int height = 24 + index * 8; + + place(chunk, 8, height, 8, Block.GLOWSTONE); + place(chunk, 2, height + 5, 12, Block.GLOWSTONE); + + for (int y = height - 6; y <= height + 6; y++) { + for (int z = 0; z < 16; z++) { + place(chunk, 11, y, z, Block.STONE); + } + } + chunks.add(chunk); + } + return chunks; + } + + /** + * Loads one chunk per worker and covers every one of them with a ceiling at its own height. + *

+ * A ceiling is what makes a sky light run interesting, because the light falls freely above it + * and has to spread step by step through the hole that is left open below it. + *

+ * + * @param instance the instance which holds the chunks + * @return the prepared chunks, one per worker + */ + private static List ceilingChunks(Instance instance) { + List chunks = new ArrayList<>(THREAD_COUNT); + + for (int index = 0; index < THREAD_COUNT; index++) { + Chunk chunk = instance.loadChunk(index, 0).join(); + int height = 40 + index * 8; + + for (int x = 0; x < 16; x++) { + for (int z = 0; z < 16; z++) { + place(chunk, x, height, z, Block.STONE); + } + } + // A single hole lets the sky light in and makes it spread underneath the ceiling. + place(chunk, 8, height, 8, Block.AIR); + chunks.add(chunk); + } + return chunks; + } + + /** + * Places a block in the given chunk while holding its write lock. + * + * @param chunk the chunk which receives the block + * @param x the x coordinate inside the chunk + * @param y the y coordinate of the block + * @param z the z coordinate inside the chunk + * @param block the block to place + */ + private static void place(Chunk chunk, int x, int y, int z, Block block) { + chunk.lockWriteLock(); + try { + chunk.setBlock(x, y, z, block); + } finally { + chunk.unlockWriteLock(); + } + } + + /** + * Reads the stored block light of every section of the given chunk. + * + * @param chunk the chunk to read + * @return the stored bytes of every section, ordered from the lowest section upwards + */ + private static List blockLightOf(Chunk chunk) { + chunk.lockReadLock(); + try { + List
sections = chunk.getSections(); + List light = new ArrayList<>(sections.size()); + + for (Section section : sections) { + light.add(section.blockLight().array()); + } + return light; + } finally { + chunk.unlockReadLock(); + } + } + + /** + * Reads the stored sky light of every section of the given chunk. + * + * @param chunk the chunk to read + * @return the stored bytes of every section, ordered from the lowest section upwards + */ + private static List skyLightOf(Chunk chunk) { + chunk.lockReadLock(); + try { + List
sections = chunk.getSections(); + List light = new ArrayList<>(sections.size()); + + for (Section section : sections) { + light.add(section.skyLight().array()); + } + return light; + } finally { + chunk.unlockReadLock(); + } + } + + /** + * Compares the light of every section against the reference. + * + * @param expected the light of every section of the single threaded run + * @param actual the light of every section which was calculated by a worker + * @param label the name of the pass which produced the light + * @param worker the number of the worker which produced the light + * @param round the round the light was calculated in + */ + private static void assertLight(List expected, List actual, String label, int worker, int round) { + assertEquals(expected.size(), actual.size(), "the " + label + " of the worker " + worker + " lost a section in round " + round); + + for (int section = 0; section < expected.size(); section++) { + assertArrayEquals( + expected.get(section), actual.get(section), + "the " + label + " of the section " + section + " of the worker " + worker + " drifted in round " + round + ); + } + } + + /** + * Waits for the given latch and fails when it is not released in time. + * + * @param latch the latch to wait for + */ + private static void awaitStart(CountDownLatch latch) { + try { + assertTrue(latch.await(AWAIT_SECONDS, TimeUnit.SECONDS), "a worker waited too long for its barrier"); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + fail("a worker was interrupted while it waited for its barrier"); + } + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/light/ChunkLightServiceIntegrationTest.java b/src/test/java/net/theevilreaper/aves/instance/light/ChunkLightServiceIntegrationTest.java new file mode 100644 index 00000000..1e6caa0d --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/light/ChunkLightServiceIntegrationTest.java @@ -0,0 +1,281 @@ +package net.theevilreaper.aves.instance.light; + +import net.minestom.server.instance.Chunk; +import net.minestom.server.instance.Instance; +import net.minestom.server.instance.block.Block; +import net.minestom.testing.Env; +import net.minestom.testing.extension.MicrotusExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests that the light engine can be applied to a chunk of a running server, independent of the + * chunk loader that produced the chunk. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ExtendWith(MicrotusExtension.class) +class ChunkLightServiceIntegrationTest { + + private final ChunkLightService service = new ChunkLightService(); + + /** + * Places a block in the given chunk while holding its write lock. + * + * @param chunk the chunk which receives the block + * @param x the x coordinate inside the chunk + * @param y the y coordinate of the block + * @param z the z coordinate inside the chunk + * @param block the block to place + */ + private static void place(Chunk chunk, int x, int y, int z, Block block) { + chunk.lockWriteLock(); + try { + chunk.setBlock(x, y, z, block); + } finally { + chunk.unlockWriteLock(); + } + } + + @Test + void testALampLightsItsSurroundings(Env env) { + Instance instance = env.createEmptyInstance(); + Chunk chunk = instance.loadChunk(0, 0).join(); + place(chunk, 8, 40, 8, Block.GLOWSTONE); + + this.service.calculate(chunk); + + assertEquals(15, this.service.blockLightAt(chunk, 8, 40, 8)); + assertEquals(14, this.service.blockLightAt(chunk, 9, 40, 8)); + assertEquals(13, this.service.blockLightAt(chunk, 10, 40, 8)); + } + + @Test + void testAChunkWithoutASourceStaysDark(Env env) { + Instance instance = env.createEmptyInstance(); + Chunk chunk = instance.loadChunk(0, 0).join(); + + this.service.calculate(chunk); + + assertEquals(0, this.service.blockLightAt(chunk, 8, 40, 8)); + } + + @Test + void testTheLightIsWrittenIntoTheSectionsOfTheChunk(Env env) { + Instance instance = env.createEmptyInstance(); + Chunk chunk = instance.loadChunk(0, 0).join(); + place(chunk, 8, 40, 8, Block.GLOWSTONE); + + this.service.calculate(chunk); + + // The engine has to hand its result to Minestom, not keep it on the side. + chunk.lockReadLock(); + try { + int sectionIndex = (40 >> 4) - chunk.getMinSection(); + byte[] stored = chunk.getSections().get(sectionIndex).blockLight().array(); + + assertTrue(stored.length > 0, "the section has to carry the calculated light"); + } finally { + chunk.unlockReadLock(); + } + } + + @Test + void testLightCrossesASectionBorderOfARealChunk(Env env) { + Instance instance = env.createEmptyInstance(); + Chunk chunk = instance.loadChunk(0, 0).join(); + // Placed at the very top of its section so the light has to reach the one above. + place(chunk, 8, 47, 8, Block.GLOWSTONE); + + this.service.calculate(chunk); + + assertEquals(15, this.service.blockLightAt(chunk, 8, 47, 8)); + assertEquals(14, this.service.blockLightAt(chunk, 8, 48, 8), "48 is the first block of the next section"); + assertEquals(13, this.service.blockLightAt(chunk, 8, 49, 8)); + } + + @Test + void testAWallStopsTheLight(Env env) { + Instance instance = env.createEmptyInstance(); + Chunk chunk = instance.loadChunk(0, 0).join(); + place(chunk, 8, 40, 8, Block.GLOWSTONE); + + // The wall has to span further than the light reaches in every direction, otherwise the + // light simply travels around it and the test proves nothing. + for (int y = 25; y <= 56; y++) { + for (int z = 0; z < 16; z++) { + place(chunk, 9, y, z, Block.STONE); + } + } + + this.service.calculate(chunk); + + assertEquals(0, this.service.blockLightAt(chunk, 10, 40, 8), "the wall has to stop the light"); + assertEquals(14, this.service.blockLightAt(chunk, 7, 40, 8), "the open side stays lit"); + } + + @Test + void testCalculatingTwiceIsStable(Env env) { + Instance instance = env.createEmptyInstance(); + Chunk chunk = instance.loadChunk(0, 0).join(); + place(chunk, 8, 40, 8, Block.GLOWSTONE); + + this.service.calculate(chunk); + this.service.calculate(chunk); + + assertEquals(14, this.service.blockLightAt(chunk, 9, 40, 8)); + } + + @Test + void testRemovingTheSourceClearsTheLight(Env env) { + Instance instance = env.createEmptyInstance(); + Chunk chunk = instance.loadChunk(0, 0).join(); + place(chunk, 8, 40, 8, Block.GLOWSTONE); + this.service.calculate(chunk); + + place(chunk, 8, 40, 8, Block.AIR); + this.service.calculate(chunk); + + assertEquals(0, this.service.blockLightAt(chunk, 9, 40, 8), "a full recalculation has to retract the light"); + } + + @Test + void testTheServiceWorksOnAChunkFromTheAnvilLoader(Env env, @org.junit.jupiter.api.io.TempDir java.nio.file.Path worldRoot) throws java.io.IOException { + net.kyori.adventure.key.Key dimension = net.kyori.adventure.key.Key.key("minecraft:overworld"); + + try (var loader = new net.theevilreaper.aves.instance.anvil.AvesAnvilLoader(worldRoot, dimension)) { + Instance instance = env.createEmptyInstance(loader); + Chunk chunk = instance.loadChunk(0, 0).join(); + place(chunk, 8, 40, 8, Block.GLOWSTONE); + loader.saveChunk(chunk); + + Chunk reloaded = loader.loadChunk(instance, 0, 0); + assertTrue(reloaded != null); + + this.service.calculate(reloaded); + + assertEquals(15, this.service.blockLightAt(reloaded, 8, 40, 8)); + assertEquals(14, this.service.blockLightAt(reloaded, 9, 40, 8)); + } + } + + @Test + void testSkyLightReachesTheGroundOfAnOpenChunk(Env env) { + Instance instance = env.createEmptyInstance(); + Chunk chunk = instance.loadChunk(0, 0).join(); + + this.service.calculateSky(chunk); + + chunk.lockReadLock(); + try { + assertEquals(15, chunk.getSectionAt(40).skyLight().getLevel(8, 40 & 15, 8), + "an open column is lit down to the bottom"); + } finally { + chunk.unlockReadLock(); + } + } + + @Test + void testACeilingKeepsTheSkyLightOut(Env env) { + Instance instance = env.createEmptyInstance(); + Chunk chunk = instance.loadChunk(0, 0).join(); + + for (int x = 0; x < 16; x++) { + for (int z = 0; z < 16; z++) { + place(chunk, x, 60, z, Block.STONE); + } + } + + this.service.calculateSky(chunk); + + chunk.lockReadLock(); + try { + assertEquals(0, chunk.getSectionAt(40).skyLight().getLevel(8, 40 & 15, 8), + "everything below the ceiling stays dark"); + } finally { + chunk.unlockReadLock(); + } + } + + @Test + void testLightCrossesIntoTheNeighbouringChunk(Env env) { + Instance instance = env.createEmptyInstance(); + Chunk west = instance.loadChunk(0, 0).join(); + Chunk east = instance.loadChunk(1, 0).join(); + // The lamp sits on the eastern edge of the western chunk. + place(west, 15, 40, 8, Block.GLOWSTONE); + + this.service.calculateWithNeighbours(instance, 0, 0); + + assertEquals(15, this.service.blockLightAt(west, 15, 40, 8)); + assertEquals(14, this.service.blockLightAt(east, 0, 40, 8), + "the first block of the neighbouring chunk has to be lit"); + assertEquals(13, this.service.blockLightAt(east, 1, 40, 8)); + } + + @Test + void testLightReachesTheChunkBehindTheNeighbour(Env env) { + Instance instance = env.createEmptyInstance(); + Chunk origin = instance.loadChunk(0, 0).join(); + Chunk east = instance.loadChunk(1, 0).join(); + Chunk south = instance.loadChunk(0, 1).join(); + Chunk diagonal = instance.loadChunk(1, 1).join(); + // The lamp sits in the corner of its chunk, so its light leaves through two borders and + // has to travel through one of the neighbours to arrive in the chunk behind them. + place(origin, 15, 40, 15, Block.GLOWSTONE); + + this.service.calculateWithNeighbours(instance, 0, 0); + + assertEquals(14, this.service.blockLightAt(east, 0, 40, 15)); + assertEquals(14, this.service.blockLightAt(south, 15, 40, 0)); + assertEquals(13, this.service.blockLightAt(diagonal, 0, 40, 0), + "the light has to continue through a neighbour into the chunk behind it"); + } + + @Test + void testARepeatedExchangeKeepsTheSameResult(Env env) { + Instance instance = env.createEmptyInstance(); + Chunk origin = instance.loadChunk(0, 0).join(); + Chunk diagonal = instance.loadChunk(1, 1).join(); + instance.loadChunk(1, 0).join(); + instance.loadChunk(0, 1).join(); + place(origin, 15, 40, 15, Block.GLOWSTONE); + + this.service.calculateWithNeighbours(instance, 0, 0); + int first = this.service.blockLightAt(diagonal, 0, 40, 0); + this.service.calculateWithNeighbours(instance, 0, 0); + + assertEquals(first, this.service.blockLightAt(diagonal, 0, 40, 0), + "a settled exchange must not drift when it is repeated"); + } + + @Test + void testTheExchangeNeverLoadsAMissingNeighbour(Env env) { + Instance instance = env.createEmptyInstance(); + Chunk origin = instance.loadChunk(0, 0).join(); + place(origin, 15, 40, 15, Block.GLOWSTONE); + + this.service.calculateWithNeighbours(instance, 0, 0); + + assertNull(instance.getChunk(1, 0), "a missing neighbour must not be loaded"); + assertNull(instance.getChunk(1, 1), "a missing diagonal neighbour must not be loaded either"); + } + + @Test + void testAMissingNeighbourIsSkipped(Env env) { + Instance instance = env.createEmptyInstance(); + Chunk chunk = instance.loadChunk(0, 0).join(); + place(chunk, 15, 40, 8, Block.GLOWSTONE); + + // Only this chunk is loaded, the neighbours are absent. + this.service.calculateWithNeighbours(instance, 0, 0); + + assertEquals(15, this.service.blockLightAt(chunk, 15, 40, 8)); + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/light/ChunkLightStateTest.java b/src/test/java/net/theevilreaper/aves/instance/light/ChunkLightStateTest.java new file mode 100644 index 00000000..8626f529 --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/light/ChunkLightStateTest.java @@ -0,0 +1,235 @@ +package net.theevilreaper.aves.instance.light; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests the incremental update of an already calculated chunk light. + *

+ * Adding a light source only spreads brightness, which a plain search handles. Removing one is the + * hard case: the light it had spread has to be retracted first, otherwise it stays behind as a glow + * without a source. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class ChunkLightStateTest { + + private static final int AIR = 0; + private static final int STONE = 1; + private static final int LAMP = 2; + + private static final BlockLightSource SOURCE = new BlockLightSource() { + + @Override + public int emission(int stateId) { + return stateId == LAMP ? 15 : 0; + } + + @Override + public boolean blocksFace(int stateId, BlockFace face) { + return stateId == STONE; + } + }; + + /** + * Builds the state ids of a chunk made of the given amount of air sections. + * + * @param sectionCount the amount of sections the chunk holds + * @return the state ids of every section + */ + private static List airChunk(int sectionCount) { + List sections = new ArrayList<>(sectionCount); + + for (int i = 0; i < sectionCount; i++) { + sections.add(new int[LightNibbles.BLOCK_COUNT]); + } + return sections; + } + + /** + * Converts the state ids of a chunk into opacity tables. + * + * @param sections the state ids of every section + * @return the opacity table of every section + */ + private static List tables(List sections) { + return sections.stream().map(states -> SectionOpacity.of(states, SOURCE)).toList(); + } + + /** + * Calculates the index of a block inside a section. + * + * @param x the x coordinate inside the section + * @param y the y coordinate inside the section + * @param z the z coordinate inside the section + * @return the index of the block + */ + private static int index(int x, int y, int z) { + return (y << 8) | (z << 4) | x; + } + + @Test + void testAFreshStateHoldsTheCalculatedLight() { + List sections = airChunk(2); + sections.get(0)[index(8, 8, 8)] = LAMP; + + ChunkLightState state = ChunkLightState.blockLight(tables(sections)); + + assertEquals(15, state.get(8, 8, 8)); + assertEquals(14, state.get(9, 8, 8)); + } + + @Test + void testAddingASourceLightsItsSurroundings() { + List sections = airChunk(2); + ChunkLightState state = ChunkLightState.blockLight(tables(sections)); + + sections.get(0)[index(8, 8, 8)] = LAMP; + state.update(tables(sections), 8, 8, 8); + + assertEquals(15, state.get(8, 8, 8)); + assertEquals(14, state.get(9, 8, 8)); + assertEquals(13, state.get(10, 8, 8)); + } + + @Test + void testRemovingASourceRetractsItsLight() { + List sections = airChunk(2); + sections.get(0)[index(8, 8, 8)] = LAMP; + ChunkLightState state = ChunkLightState.blockLight(tables(sections)); + + sections.get(0)[index(8, 8, 8)] = AIR; + state.update(tables(sections), 8, 8, 8); + + assertEquals(0, state.get(8, 8, 8), "the source itself has to go dark"); + assertEquals(0, state.get(9, 8, 8), "the light it had spread has to be retracted"); + assertEquals(0, state.get(12, 8, 8)); + } + + @Test + void testRemovingOneOfTwoSourcesKeepsTheOther() { + List sections = airChunk(2); + sections.get(0)[index(4, 8, 8)] = LAMP; + sections.get(0)[index(12, 8, 8)] = LAMP; + ChunkLightState state = ChunkLightState.blockLight(tables(sections)); + + sections.get(0)[index(4, 8, 8)] = AIR; + state.update(tables(sections), 4, 8, 8); + + assertEquals(15, state.get(12, 8, 8), "the remaining source keeps its level"); + assertEquals(14, state.get(11, 8, 8)); + assertEquals(7, state.get(4, 8, 8), "the removed position is refilled from the other source"); + } + + @Test + void testTheIncrementalResultMatchesAFullRecalculation() { + List sections = airChunk(2); + sections.get(0)[index(4, 8, 8)] = LAMP; + sections.get(0)[index(12, 8, 8)] = LAMP; + ChunkLightState state = ChunkLightState.blockLight(tables(sections)); + + sections.get(0)[index(4, 8, 8)] = AIR; + state.update(tables(sections), 4, 8, 8); + + ChunkLightState fresh = ChunkLightState.blockLight(tables(sections)); + + for (int y = 0; y < 32; y++) { + for (int z = 0; z < 16; z++) { + for (int x = 0; x < 16; x++) { + assertEquals(fresh.get(x, y, z), state.get(x, y, z), + "mismatch at " + x + "/" + y + "/" + z); + } + } + } + } + + @Test + void testPlacingABlockingBlockRemovesTheLightBehindIt() { + List sections = airChunk(2); + sections.get(0)[index(8, 8, 8)] = LAMP; + ChunkLightState state = ChunkLightState.blockLight(tables(sections)); + + sections.get(0)[index(9, 8, 8)] = STONE; + state.update(tables(sections), 9, 8, 8); + + assertEquals(0, state.get(9, 8, 8), "the new block itself carries no light"); + ChunkLightState fresh = ChunkLightState.blockLight(tables(sections)); + assertEquals(fresh.get(10, 8, 8), state.get(10, 8, 8)); + } + + @Test + void testAnUpdateCrossesTheSectionBorder() { + List sections = airChunk(2); + ChunkLightState state = ChunkLightState.blockLight(tables(sections)); + + sections.get(0)[index(8, 15, 8)] = LAMP; + state.update(tables(sections), 8, 15, 8); + + assertEquals(14, state.get(8, 16, 8), "16 is the first block of the next section"); + } + + @Test + void testTheStateCanBeReadAsSections() { + List sections = airChunk(2); + sections.get(0)[index(8, 8, 8)] = LAMP; + ChunkLightState state = ChunkLightState.blockLight(tables(sections)); + + List light = state.toSections(); + + assertEquals(2, light.size()); + assertEquals(15, light.get(0).get(8, 8, 8)); + assertEquals(14, light.get(0).get(9, 8, 8)); + } + + @Test + void testASkyLightStateFallsStraightDown() { + ChunkLightState state = ChunkLightState.skyLight(tables(airChunk(2))); + + assertEquals(15, state.get(8, 0, 8)); + assertEquals(15, state.get(8, 31, 8)); + } + + @Test + void testAnUpdateWithManyGradedSourcesDoesNotOverflowTheQueue() { + // Same hazard as in the propagators: a position enters the addition queue again whenever a + // brighter source raises it, so a queue sized for one entry per position is too small. + BlockLightSource graded = new BlockLightSource() { + + @Override + public int emission(int stateId) { + return stateId == 0 ? 0 : stateId; + } + + @Override + public boolean blocksFace(int stateId, BlockFace face) { + return false; + } + }; + + List sections = airChunk(2); + + for (int y = 0; y < LightNibbles.DIMENSION; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + sections.get(0)[index(x, y, z)] = 1 + ((x + y + z) % 3); + } + } + } + + List tables = sections.stream().map(states -> SectionOpacity.of(states, graded)).toList(); + ChunkLightState state = ChunkLightState.blockLight(tables); + + sections.get(1)[index(8, 8, 8)] = 15; + List updated = sections.stream().map(states -> SectionOpacity.of(states, graded)).toList(); + state.update(updated, 8, 24, 8); + + assertEquals(15, state.get(8, 24, 8)); + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/light/LightEngineConcurrencyTest.java b/src/test/java/net/theevilreaper/aves/instance/light/LightEngineConcurrencyTest.java new file mode 100644 index 00000000..b2f4326e --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/light/LightEngineConcurrencyTest.java @@ -0,0 +1,436 @@ +package net.theevilreaper.aves.instance.light; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Stresses the parts of the light engine which are used from more than one thread. + *

+ * The engine splits its types into two groups. {@link SectionOpacity} is immutable once it is built + * and is therefore shared between the workers of a chunk batch, while {@link LightPropagator} and + * {@link ChunkLightPropagator} keep working buffers and are documented as reusable but confined to + * a single thread. The tests here verify exactly that split: many threads may read one opacity + * table, and many threads may propagate at the same time as long as every one of them owns its + * propagator. + *

+ *

+ * Sharing a single propagator between threads is deliberately not tested. It is not part of the + * contract of those types, their buffers are plain arrays which are cleared at the start of a run, + * and a test which asserted anything about that case would only pin down undefined behaviour. The + * contract that is worth protecting is the one this class asserts, namely that an independent + * instance per thread produces the same result as a single threaded run. + *

+ *

+ * {@link ChunkLightService} is the other side of that split and does promise to serve many threads + * at once, which it can only do by giving every call its own propagator. + * {@link ChunkLightServiceConcurrencyTest} holds it to that promise. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class LightEngineConcurrencyTest { + + /** + * The time a latch is waited for before the test is considered stuck. + */ + private static final long AWAIT_SECONDS = 60L; + + /** + * The state id of a block which neither emits nor occludes light. + */ + private static final int AIR = 0; + + /** + * The state id of a block which occludes every face. + */ + private static final int STONE = 1; + + /** + * The state id of a block which emits the highest level. + */ + private static final int LAMP = 2; + + /** + * The state id of a block which occludes its lower face only. + */ + private static final int SLAB = 3; + + /** + * A source which describes the four blocks of the fixtures without touching any registry. + *

+ * Every emitting block of the fixtures carries the same level on purpose. The breadth first + * search of the engine assumes that the queued positions are ordered by their level, which only + * holds while every source starts at the same one. Mixing two emission levels makes the search + * revisit positions and overflow its queue, which is a defect of the propagation itself and has + * nothing to do with the thread safety these tests are about. + *

+ */ + private static final BlockLightSource SOURCE = new BlockLightSource() { + + @Override + public int emission(int stateId) { + return stateId == LAMP ? 15 : 0; + } + + @Override + public boolean blocksFace(int stateId, BlockFace face) { + return switch (stateId) { + case STONE -> true; + case SLAB -> face == BlockFace.BOTTOM; + default -> false; + }; + } + }; + + @Test + void testIndependentPropagatorsProduceTheSameResultAsASingleThreadedRun() throws InterruptedException, ExecutionException { + // The reference is calculated on one thread first. Any state which leaks out of a propagator + // into a shared place, a static buffer for example, makes the parallel results drift away + // from that reference. Every thread runs every fixture repeatedly so a result which only + // breaks under interleaving is hit as well. + List fixtures = sections(6); + List expected = new ArrayList<>(fixtures.size()); + List expectedUniform = new ArrayList<>(fixtures.size()); + LightPropagator reference = new LightPropagator(); + + for (SectionOpacity fixture : fixtures) { + LightNibbles light = reference.propagate(fixture); + expected.add(light.toDenseArray()); + expectedUniform.add(light.isUniform()); + } + + int threadCount = 8; + int rounds = 12; + CountDownLatch start = new CountDownLatch(1); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(threadCount); + + for (int thread = 0; thread < threadCount; thread++) { + futures.add(executor.submit(() -> { + LightPropagator propagator = new LightPropagator(); + awaitStart(start); + + for (int round = 0; round < rounds; round++) { + for (int fixture = 0; fixture < fixtures.size(); fixture++) { + LightNibbles light = propagator.propagate(fixtures.get(fixture)); + + assertArrayEquals(expected.get(fixture), light.toDenseArray(), "the fixture " + fixture + " drifted in round " + round); + assertEquals(expectedUniform.get(fixture), light.isUniform(), "the fixture " + fixture + " changed its storage in round " + round); + } + } + return null; + })); + } + start.countDown(); + awaitAll(futures); + } + } + + @Test + void testIndependentChunkPropagatorsProduceTheSameResultAsASingleThreadedRun() throws InterruptedException, ExecutionException { + // The chunk propagator sizes its buffers on the first run and keeps them afterwards, which + // is the part that would break loudly if an instance were shared. Every thread owns one, so + // both the block light and the sky light of the same column have to match the reference. + List column = sections(4); + ChunkLightPropagator reference = new ChunkLightPropagator(); + List expectedBlock = dense(reference.propagate(column)); + List expectedSky = dense(reference.propagateSky(column)); + + int threadCount = 8; + int rounds = 8; + CountDownLatch start = new CountDownLatch(1); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(threadCount); + + for (int thread = 0; thread < threadCount; thread++) { + futures.add(executor.submit(() -> { + ChunkLightPropagator propagator = new ChunkLightPropagator(); + awaitStart(start); + + for (int round = 0; round < rounds; round++) { + assertDense(expectedBlock, propagator.propagate(column), "block light", round); + assertDense(expectedSky, propagator.propagateSky(column), "sky light", round); + } + return null; + })); + } + start.countDown(); + awaitAll(futures); + } + } + + @Test + void testConcurrentReadsOfAnOpacityTableStayConsistent() throws InterruptedException, ExecutionException { + // The table is built once and shared by every worker of a chunk batch, so it has to answer + // the same thing to every thread forever. A checksum over every position and every face + // covers the whole table in one value, so a single entry which changed under concurrency + // makes the checksum of that thread differ from the reference. + SectionOpacity opacity = sections(1).getFirst(); + long expected = checksum(opacity); + boolean expectedEmission = opacity.hasEmission(); + boolean expectedTransparency = opacity.isFullyTransparent(); + + int threadCount = 16; + int rounds = 20; + CountDownLatch start = new CountDownLatch(1); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(threadCount); + + for (int thread = 0; thread < threadCount; thread++) { + futures.add(executor.submit(() -> { + awaitStart(start); + + for (int round = 0; round < rounds; round++) { + assertEquals(expected, checksum(opacity), "the opacity table answered differently in round " + round); + assertEquals(expectedEmission, opacity.hasEmission()); + assertEquals(expectedTransparency, opacity.isFullyTransparent()); + } + return null; + })); + } + start.countDown(); + awaitAll(futures); + } + } + + @Test + void testACopyStaysIndependentWhileItsSourceIsMutated() throws InterruptedException, ExecutionException { + // A propagation hands its result to another thread through a copy, so a copy which still + // shared the array of its source would let the next mutation of the source rewrite a result + // that was already published. One thread rewrites the source continuously while the others + // keep reading the copy, which turns a shared array into an immediate mismatch. + LightNibbles source = LightNibbles.uniform(0); + + for (int y = 0; y < LightNibbles.DIMENSION; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + source.set(x, y, z, (x + y + z) % 8); + } + } + } + + LightNibbles copy = source.copy(); + byte[] expected = copy.toDenseArray(); + int readerCount = 8; + int rounds = 200; + CountDownLatch start = new CountDownLatch(1); + CountDownLatch readersDone = new CountDownLatch(readerCount); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(readerCount + 1); + + for (int reader = 0; reader < readerCount; reader++) { + futures.add(executor.submit(() -> { + awaitStart(start); + + try { + for (int round = 0; round < rounds; round++) { + assertArrayEquals(expected, copy.toDenseArray(), "the copy changed in round " + round); + assertFalse(copy.isUniform(), "the copy must keep its own array"); + assertEquals(0, copy.get(0, 0, 0), "the copy must not see the mutation of its source"); + } + } finally { + readersDone.countDown(); + } + return null; + })); + } + + // A single writer is enough and keeps the source itself well defined. The class is + // documented as not thread safe, so mutating it from several threads at once would only + // race with itself instead of testing the independence of the copy. + futures.add(executor.submit(() -> { + awaitStart(start); + + while (readersDone.getCount() > 0) { + for (int y = 0; y < LightNibbles.DIMENSION; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + source.set(x, y, z, LightNibbles.MAX_LEVEL); + } + } + } + } + return null; + })); + start.countDown(); + awaitAll(futures); + } + + assertArrayEquals(expected, copy.toDenseArray(), "the copy has to survive every mutation of its source"); + assertEquals(LightNibbles.MAX_LEVEL, source.get(0, 0, 0), "the source has to carry the mutation"); + assertEquals(0, copy.get(0, 0, 0), "the copy has to keep the value it was created with"); + } + + @Test + void testACopyOfAUniformSectionStaysUniformWhileItsSourceGrowsAnArray() throws InterruptedException, ExecutionException { + // A uniform section carries no array at all. Its copy has to stay uniform even though the + // source allocates one the moment a differing level is written into it. + LightNibbles source = LightNibbles.uniform(7); + LightNibbles copy = source.copy(); + int readerCount = 8; + int rounds = 500; + CountDownLatch start = new CountDownLatch(1); + CountDownLatch readersDone = new CountDownLatch(readerCount); + + try (ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor()) { + List> futures = new ArrayList<>(readerCount + 1); + + for (int reader = 0; reader < readerCount; reader++) { + futures.add(executor.submit(() -> { + awaitStart(start); + + try { + for (int round = 0; round < rounds; round++) { + assertTrue(copy.isUniform(), "the copy of a uniform section must not grow an array"); + assertEquals(7, copy.get(3, 4, 5), "the copy has to keep its uniform level"); + } + } finally { + readersDone.countDown(); + } + return null; + })); + } + + futures.add(executor.submit(() -> { + awaitStart(start); + + while (readersDone.getCount() > 0) { + source.set(3, 4, 5, 1); + source.fill(7); + } + return null; + })); + start.countDown(); + awaitAll(futures); + } + + assertTrue(copy.isUniform()); + assertEquals(7, copy.get(3, 4, 5)); + } + + /** + * Builds the given amount of sections with a repeatable pseudo random content. + * The seed is fixed so every run of the test works on the same fixtures. + * + * @param count the amount of sections to build + * @return the created sections + */ + private static List sections(int count) { + Random random = new Random(0x5EEDL); + List sections = new ArrayList<>(count); + int[] palette = {AIR, AIR, AIR, AIR, STONE, STONE, SLAB, SLAB, LAMP}; + + for (int section = 0; section < count; section++) { + int[] states = new int[LightNibbles.BLOCK_COUNT]; + + for (int index = 0; index < states.length; index++) { + states[index] = palette[random.nextInt(palette.length)]; + } + sections.add(SectionOpacity.of(states, SOURCE)); + } + return sections; + } + + /** + * Converts the light of every section into its dense byte representation. + * + * @param light the light of every section + * @return the dense bytes of every section + */ + private static List dense(List light) { + List arrays = new ArrayList<>(light.size()); + + for (LightNibbles section : light) { + arrays.add(section.toDenseArray()); + } + return arrays; + } + + /** + * Compares the light of every section against the expected bytes. + * + * @param expected the expected bytes of every section + * @param actual the light which was calculated + * @param label the name of the pass which produced the light + * @param round the round the light was calculated in + */ + private static void assertDense(List expected, List actual, String label, int round) { + assertEquals(expected.size(), actual.size(), "the " + label + " lost a section in round " + round); + + for (int section = 0; section < expected.size(); section++) { + assertArrayEquals(expected.get(section), actual.get(section).toDenseArray(), "the " + label + " of the section " + section + " drifted in round " + round); + } + } + + /** + * Folds every answer of the given table into a single value. + * A table which answers differently for a single position or face changes the result. + * + * @param opacity the table to read + * @return the checksum of the table + */ + private static long checksum(SectionOpacity opacity) { + long value = 0L; + + for (int y = 0; y < LightNibbles.DIMENSION; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + value = value * 31L + opacity.emission(x, y, z); + + for (BlockFace face : BlockFace.values()) { + value = value * 31L + (opacity.blocksFace(x, y, z, face) ? 1L : 0L); + } + } + } + } + return value; + } + + /** + * Waits for the given latch and fails when it is not released in time. + * + * @param latch the latch to wait for + */ + private static void awaitStart(CountDownLatch latch) { + try { + assertTrue(latch.await(AWAIT_SECONDS, TimeUnit.SECONDS), "a worker waited too long for its barrier"); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + fail("a worker was interrupted while it waited for its barrier"); + } + } + + /** + * Waits for every given task and propagates the failure of the first broken one. + * + * @param futures the tasks to wait for + * @throws InterruptedException if the waiting thread is interrupted + * @throws ExecutionException if a task failed + */ + private static void awaitAll(List> futures) throws InterruptedException, ExecutionException { + for (Future future : futures) { + future.get(); + } + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/light/LightEngineEquivalenceTest.java b/src/test/java/net/theevilreaper/aves/instance/light/LightEngineEquivalenceTest.java new file mode 100644 index 00000000..a408a4c9 --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/light/LightEngineEquivalenceTest.java @@ -0,0 +1,174 @@ +package net.theevilreaper.aves.instance.light; + +import net.minestom.server.instance.block.Block; +import net.minestom.server.instance.palette.Palette; +import net.minestom.testing.extension.MicrotusExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import java.lang.reflect.Method; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Pins down that the light engine of Aves produces the very same bytes as the one the server ships + * with, on every section both of them are given. + *

+ * This is the constraint every optimisation of the engine has to survive. A faster engine which + * lights a section even slightly differently is not a faster engine, it is a second lighting of the + * same world, and the difference would surface as a patch of wrong brightness that no player can + * explain and no log mentions. The engines are therefore compared byte for byte and not by any + * weaker notion of similarity. + *

+ *

+ * The two methods which form the built-in path, {@code BlockLight.buildInternalQueue} and + * {@code LightCompute.compute}, are package-private. They are called through reflection rather than + * from a test placed inside the Minestom package, because that placement would split a package of + * the server across two artifacts and would drag the queue type of the built-in path onto the test + * classpath as a compile dependency. Reflection keeps the comparison to the one thing it is about: + * the bytes that come out of each engine. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ExtendWith(MicrotusExtension.class) +class LightEngineEquivalenceTest { + + /** + * The amounts of light sources a compared section is filled with. + */ + private static final int[] LIGHT_SOURCES = {0, 1, 2, 4, 8, 16, 64, 128, 512}; + + /** + * The shares of solid blocks a compared section is filled with, in percent. + */ + private static final int[] OCCLUSION_PERCENTS = {0, 10, 30, 50, 70, 90}; + + /** + * The seed every section is built from, so a failure can be reproduced. + */ + private static final long SEED = 20260731L; + + /** + * The amount of bytes a light section occupies. + */ + private static final int LIGHT_LENGTH = LightNibbles.ARRAY_LENGTH; + + @Test + void testBothEnginesLightEverySectionIdentically() { + MinestomBlockLightSource source = new MinestomBlockLightSource(); + LightPropagator propagator = new LightPropagator(); + int compared = 0; + + for (int lightSources : LIGHT_SOURCES) { + for (int occlusionPercent : OCCLUSION_PERCENTS) { + int[] states = section(lightSources, occlusionPercent); + Palette palette = paletteOf(states); + + byte[] expected = minestomLight(palette); + byte[] actual = propagator.propagate(SectionOpacity.of(states, source)).toDenseArray(); + + assertArrayEquals( + expected, actual, + "the engines disagree on a section with " + lightSources + " sources and " + + occlusionPercent + " percent solid blocks" + ); + + // A comparison of two dark sections would agree no matter what either engine does, + // so every scenario which holds a source has to carry light to be worth anything. + if (lightSources > 0) { + assertFalse( + java.util.Arrays.equals(new byte[LIGHT_LENGTH], actual), + "a section with " + lightSources + " sources and " + occlusionPercent + + " percent solid blocks stayed dark, so it compares nothing" + ); + } + compared++; + } + } + assertEquals(LIGHT_SOURCES.length * OCCLUSION_PERCENTS.length, compared); + } + + @Test + void testBothEnginesAgreeOnASectionWithoutAnyLight() { + // The engines store an unlit section differently, one as an empty array and the other + // without an array at all. Both have to hand out the same bytes regardless. + int[] states = section(0, 100); + byte[] expected = minestomLight(paletteOf(states)); + byte[] actual = new LightPropagator() + .propagate(SectionOpacity.of(states, new MinestomBlockLightSource())) + .toDenseArray(); + + assertArrayEquals(new byte[LIGHT_LENGTH], actual); + assertArrayEquals(expected, actual); + } + + /** + * Builds the state ids of a section from the amount of sources and the share of solid blocks. + * + * @param lightSources the amount of light emitting blocks the section holds + * @param occlusionPercent the share of solid blocks in the section, in percent + * @return the state id of every block of the section + */ + private static int[] section(int lightSources, int occlusionPercent) { + int[] states = new int[LightNibbles.BLOCK_COUNT]; + Random random = new Random(SEED); + int air = Block.AIR.stateId(); + int stone = Block.STONE.stateId(); + int glowstone = Block.GLOWSTONE.stateId(); + + for (int index = 0; index < states.length; index++) { + states[index] = random.nextInt(100) < occlusionPercent ? stone : air; + } + for (int placed = 0; placed < lightSources; placed++) { + states[random.nextInt(states.length)] = glowstone; + } + return states; + } + + /** + * Builds the block palette of a section from its state ids. + * + * @param states the state id of every block of the section + * @return the created palette + */ + private static Palette paletteOf(int[] states) { + Palette palette = Palette.blocks(); + palette.setAll((x, y, z) -> states[(y << 8) | (z << 4) | x]); + return palette; + } + + /** + * Runs the light engine the server ships with over the given palette. + *

+ * An unlit section is reported as an empty array by that engine, which is how the network format + * encodes it. It is expanded here so both engines are compared on arrays of the same length. + *

+ * + * @param palette the block palette of the section + * @return the calculated light of the section + */ + private static byte[] minestomLight(Palette palette) { + try { + Class blockLight = Class.forName("net.minestom.server.instance.light.BlockLight"); + Class lightCompute = Class.forName("net.minestom.server.instance.light.LightCompute"); + Class queueType = Class.forName("it.unimi.dsi.fastutil.shorts.ShortArrayFIFOQueue"); + + Method buildQueue = blockLight.getDeclaredMethod("buildInternalQueue", Palette.class); + Method compute = lightCompute.getDeclaredMethod("compute", Palette.class, queueType); + buildQueue.setAccessible(true); + compute.setAccessible(true); + + byte[] light = (byte[]) compute.invoke(null, palette, buildQueue.invoke(null, palette)); + return light.length == LIGHT_LENGTH ? light : new byte[LIGHT_LENGTH]; + } catch (ReflectiveOperationException exception) { + return fail("the light engine of the server could not be reached", exception); + } + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/light/LightNibblesTest.java b/src/test/java/net/theevilreaper/aves/instance/light/LightNibblesTest.java new file mode 100644 index 00000000..3841cc4e --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/light/LightNibblesTest.java @@ -0,0 +1,271 @@ +package net.theevilreaper.aves.instance.light; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.random.RandomGenerator; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the nibble storage of a light section. Two light levels share one byte, and a section + * whose levels are all equal carries no array at all. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class LightNibblesTest { + + @Test + void testAUniformSectionCarriesNoArray() { + LightNibbles nibbles = LightNibbles.uniform(15); + + assertTrue(nibbles.isUniform()); + assertEquals(15, nibbles.get(0, 0, 0)); + assertEquals(15, nibbles.get(15, 15, 15)); + } + + @Test + void testAUniformSectionReportsTheEmptyArrayWithoutAllocating() { + LightNibbles nibbles = LightNibbles.uniform(0); + + assertEquals(0, nibbles.toArray().length, "a fully dark section needs no bytes on disk"); + } + + @Test + void testAUniformSectionExpandsIntoAFullArrayWhenAskedFor() { + byte[] array = LightNibbles.uniform(15).toDenseArray(); + + assertEquals(LightNibbles.ARRAY_LENGTH, array.length); + for (byte value : array) { + assertEquals((byte) 0xFF, value, "every nibble of the byte has to carry the level"); + } + } + + @Test + void testWritingADifferentLevelBreaksTheUniformState() { + LightNibbles nibbles = LightNibbles.uniform(0); + nibbles.set(1, 2, 3, 7); + + assertFalse(nibbles.isUniform()); + assertEquals(7, nibbles.get(1, 2, 3)); + assertEquals(0, nibbles.get(1, 2, 4)); + } + + @Test + void testWritingTheSameLevelKeepsTheUniformState() { + LightNibbles nibbles = LightNibbles.uniform(4); + nibbles.set(1, 2, 3, 4); + + assertTrue(nibbles.isUniform(), "writing the value it already holds must not allocate"); + } + + @Test + void testTwoLevelsShareOneByte() { + LightNibbles nibbles = LightNibbles.uniform(0); + nibbles.set(0, 0, 0, 1); + nibbles.set(1, 0, 0, 2); + + byte[] array = nibbles.toArray(); + + assertEquals(LightNibbles.ARRAY_LENGTH, array.length); + assertEquals(1, array[0] & 0x0F); + assertEquals(2, (array[0] >> 4) & 0x0F); + } + + @Test + void testEveryPositionIsAddressedSeparately() { + LightNibbles nibbles = LightNibbles.uniform(0); + RandomGenerator random = RandomGenerator.getDefault(); + int[][][] expected = new int[16][16][16]; + + for (int x = 0; x < 16; x++) { + for (int y = 0; y < 16; y++) { + for (int z = 0; z < 16; z++) { + int level = random.nextInt(16); + expected[x][y][z] = level; + nibbles.set(x, y, z, level); + } + } + } + + for (int x = 0; x < 16; x++) { + for (int y = 0; y < 16; y++) { + for (int z = 0; z < 16; z++) { + assertEquals(expected[x][y][z], nibbles.get(x, y, z), "mismatch at " + x + "/" + y + "/" + z); + } + } + } + } + + @Test + void testAStoredArrayIsReadBackIdentically() { + byte[] source = new byte[LightNibbles.ARRAY_LENGTH]; + RandomGenerator.getDefault().nextBytes(source); + + LightNibbles nibbles = LightNibbles.of(source); + + assertArrayEquals(source, nibbles.toArray()); + } + + @Test + void testAnArrayOfOnlyOneLevelIsRecognisedAsUniform() { + byte[] source = new byte[LightNibbles.ARRAY_LENGTH]; + java.util.Arrays.fill(source, (byte) 0x88); + + LightNibbles nibbles = LightNibbles.of(source); + + assertTrue(nibbles.isUniform(), "an array of a single repeated level should not be kept"); + assertEquals(8, nibbles.get(5, 5, 5)); + } + + @Test + void testTheStoredArrayIsCopiedOnRead() { + LightNibbles nibbles = LightNibbles.uniform(0); + nibbles.set(0, 0, 0, 5); + + assertNotSame(nibbles.toArray(), nibbles.toArray(), "callers must not be able to mutate the storage"); + } + + @Test + void testAnArrayOfTheWrongLengthIsRejected() { + assertThrows(IllegalArgumentException.class, () -> LightNibbles.of(new byte[100])); + } + + @ParameterizedTest + @ValueSource(ints = {-1, 16}) + void testALevelOutsideTheRangeIsRejected(int level) { + LightNibbles nibbles = LightNibbles.uniform(0); + + assertThrows(IllegalArgumentException.class, () -> nibbles.set(0, 0, 0, level)); + } + + @Test + void testTheUniformLevelIsRejectedWhenOutsideTheRange() { + assertThrows(IllegalArgumentException.class, () -> LightNibbles.uniform(16)); + } + + @Test + void testFillResetsTheSectionToASingleLevel() { + LightNibbles nibbles = LightNibbles.uniform(0); + nibbles.set(3, 3, 3, 9); + nibbles.fill(2); + + assertTrue(nibbles.isUniform(), "filling has to release the array again"); + assertEquals(2, nibbles.get(3, 3, 3)); + } + + @Test + void testACopyIsIndependentOfItsSource() { + LightNibbles source = LightNibbles.uniform(0); + source.set(1, 1, 1, 3); + + LightNibbles copy = source.copy(); + copy.set(1, 1, 1, 9); + + assertEquals(3, source.get(1, 1, 1)); + assertEquals(9, copy.get(1, 1, 1)); + } + + @Test + void testCopyingAUniformSectionSharesNoArray() { + LightNibbles copy = LightNibbles.uniform(7).copy(); + + assertTrue(copy.isUniform()); + assertEquals(7, copy.get(0, 0, 0)); + } + + @Test + void testTheMaximumLevelIsStoredWithoutSignIssues() { + LightNibbles nibbles = LightNibbles.uniform(0); + nibbles.set(2, 2, 2, 15); + + assertEquals(15, nibbles.get(2, 2, 2), "a nibble is unsigned, 15 must not read back as -1"); + } + + @Test + void testALevelPerPositionIsPackedIntoNibbles() { + byte[] levels = new byte[LightNibbles.BLOCK_COUNT]; + + for (int index = 0; index < levels.length; index++) { + levels[index] = (byte) (index % 16); + } + LightNibbles nibbles = LightNibbles.ofLevels(levels, 0); + + assertFalse(nibbles.isUniform()); + + for (int index = 0; index < levels.length; index++) { + assertEquals(levels[index], nibbles.get(index & 15, index >> 8, (index >> 4) & 15)); + } + } + + @Test + void testPackingAndSettingProduceTheSameBytes() { + // The packer replaces a loop which wrote every position through set. Both have to end in + // exactly the same bytes, because those bytes go to a client. + RandomGenerator random = RandomGenerator.getDefault(); + byte[] levels = new byte[LightNibbles.BLOCK_COUNT]; + LightNibbles written = LightNibbles.uniform(0); + + for (int index = 0; index < levels.length; index++) { + int level = random.nextInt(16); + levels[index] = (byte) level; + + if (level != 0) { + written.set(index & 15, index >> 8, (index >> 4) & 15, level); + } + } + + assertArrayEquals(written.toDenseArray(), LightNibbles.ofLevels(levels, 0).toDenseArray()); + } + + @Test + void testALevelArrayOfOneRepeatedLevelNeedsNoArray() { + byte[] levels = new byte[LightNibbles.BLOCK_COUNT]; + java.util.Arrays.fill(levels, (byte) 7); + + LightNibbles nibbles = LightNibbles.ofLevels(levels, 0); + + assertTrue(nibbles.isUniform()); + assertEquals(7, nibbles.get(3, 9, 12)); + } + + @Test + void testPackingReadsTheSectionWhichStartsAtTheGivenOffset() { + // A whole chunk column keeps the levels of all of its sections in one array, so a section + // has to be packed out of the middle of it. + byte[] levels = new byte[LightNibbles.BLOCK_COUNT * 3]; + java.util.Arrays.fill(levels, LightNibbles.BLOCK_COUNT, LightNibbles.BLOCK_COUNT * 2, (byte) 4); + + LightNibbles second = LightNibbles.ofLevels(levels, LightNibbles.BLOCK_COUNT); + + assertTrue(second.isUniform()); + assertEquals(4, second.get(0, 0, 0)); + assertEquals(0, LightNibbles.ofLevels(levels, 0).get(0, 0, 0)); + } + + @Test + void testALevelOutsideTheAllowedRangeIsRejected() { + byte[] levels = new byte[LightNibbles.BLOCK_COUNT]; + levels[77] = 42; + + assertThrows(IllegalArgumentException.class, () -> LightNibbles.ofLevels(levels, 0)); + } + + @Test + void testALevelArrayWhichIsTooShortIsRejected() { + assertThrows(IllegalArgumentException.class, () -> LightNibbles.ofLevels(new byte[10], 0)); + assertThrows( + IllegalArgumentException.class, + () -> LightNibbles.ofLevels(new byte[LightNibbles.BLOCK_COUNT], 1) + ); + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/light/LightPropagatorTest.java b/src/test/java/net/theevilreaper/aves/instance/light/LightPropagatorTest.java new file mode 100644 index 00000000..4351571f --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/light/LightPropagatorTest.java @@ -0,0 +1,287 @@ +package net.theevilreaper.aves.instance.light; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the light propagation inside a single section. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class LightPropagatorTest { + + private static final int AIR = 0; + private static final int STONE = 1; + private static final int LAMP = 2; + private static final int SLAB = 3; + + private static final BlockLightSource SOURCE = new BlockLightSource() { + + @Override + public int emission(int stateId) { + return stateId == LAMP ? 15 : 0; + } + + @Override + public boolean blocksFace(int stateId, BlockFace face) { + return switch (stateId) { + case STONE -> true; + case SLAB -> face == BlockFace.BOTTOM; + default -> false; + }; + } + }; + + /** + * Builds a section of air with the given blocks placed into it. + * + * @param placements triples of index and state id + * @return the state ids of the section + */ + private static int[] section(int... placements) { + int[] states = new int[LightNibbles.BLOCK_COUNT]; + + for (int i = 0; i < placements.length; i += 2) { + states[placements[i]] = placements[i + 1]; + } + return states; + } + + /** + * Calculates the index of a block inside a section. + * + * @param x the x coordinate inside the section + * @param y the y coordinate inside the section + * @param z the z coordinate inside the section + * @return the index of the block + */ + private static int index(int x, int y, int z) { + return (y << 8) | (z << 4) | x; + } + + @Test + void testASectionWithoutAnySourceStaysDark() { + SectionOpacity opacity = SectionOpacity.of(section(), SOURCE); + + LightNibbles light = new LightPropagator().propagate(opacity); + + assertTrue(light.isUniform(), "a dark section must not allocate an array"); + assertEquals(0, light.get(0, 0, 0)); + } + + @Test + void testALightSourceKeepsItsOwnLevel() { + SectionOpacity opacity = SectionOpacity.of(section(index(8, 8, 8), LAMP), SOURCE); + + LightNibbles light = new LightPropagator().propagate(opacity); + + assertEquals(15, light.get(8, 8, 8)); + } + + @Test + void testTheLevelDropsByOnePerBlock() { + SectionOpacity opacity = SectionOpacity.of(section(index(8, 8, 8), LAMP), SOURCE); + + LightNibbles light = new LightPropagator().propagate(opacity); + + assertEquals(14, light.get(9, 8, 8)); + assertEquals(13, light.get(10, 8, 8)); + assertEquals(12, light.get(11, 8, 8)); + } + + @Test + void testTheLevelSpreadsInEveryDirection() { + SectionOpacity opacity = SectionOpacity.of(section(index(8, 8, 8), LAMP), SOURCE); + + LightNibbles light = new LightPropagator().propagate(opacity); + + assertEquals(14, light.get(7, 8, 8)); + assertEquals(14, light.get(8, 7, 8)); + assertEquals(14, light.get(8, 9, 8)); + assertEquals(14, light.get(8, 8, 7)); + assertEquals(14, light.get(8, 8, 9)); + } + + @Test + void testTheLevelUsesTheShortestPath() { + SectionOpacity opacity = SectionOpacity.of(section(index(8, 8, 8), LAMP), SOURCE); + + LightNibbles light = new LightPropagator().propagate(opacity); + + // Diagonal neighbours are reached over two steps, so they lose two levels. + assertEquals(13, light.get(9, 9, 8)); + assertEquals(12, light.get(9, 9, 9)); + } + + @Test + void testTheLevelNeverFallsBelowZero() { + SectionOpacity opacity = SectionOpacity.of(section(index(0, 0, 0), LAMP), SOURCE); + + LightNibbles light = new LightPropagator().propagate(opacity); + + assertEquals(0, light.get(15, 15, 15), "a corner too far away stays dark"); + } + + @Test + void testAnOpaqueBlockStopsTheLight() { + int[] states = section(index(8, 8, 8), LAMP); + + for (int y = 0; y < LightNibbles.DIMENSION; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + states[index(9, y, z)] = STONE; + } + } + SectionOpacity opacity = SectionOpacity.of(states, SOURCE); + + LightNibbles light = new LightPropagator().propagate(opacity); + + assertEquals(0, light.get(9, 8, 8), "an opaque block carries no light"); + assertEquals(0, light.get(10, 8, 8), "the wall must block everything behind it"); + assertEquals(14, light.get(7, 8, 8), "the other side stays lit"); + } + + @Test + void testADirectionalBlockStopsLightThroughItsBlockedFace() { + // A closed layer of slabs. Each of them blocks its bottom face only, so light from below + // cannot enter the layer at all. A single slab would not prove this, because light would + // simply travel around it. + int[] states = section(index(8, 6, 8), LAMP); + + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + states[index(x, 7, z)] = SLAB; + } + } + SectionOpacity opacity = SectionOpacity.of(states, SOURCE); + + LightNibbles light = new LightPropagator().propagate(opacity); + + assertEquals(0, light.get(8, 7, 8), "light from below must not pass the blocked face"); + assertEquals(0, light.get(8, 8, 8), "nothing above the layer may be lit"); + assertEquals(14, light.get(8, 6, 9), "the level below the layer stays lit"); + } + + @Test + void testADirectionalBlockLetsLightPassItsOpenFaces() { + // The same slab layer, but the source sits inside it. The sides are open, so light spreads + // horizontally even though the bottom faces are blocked. + int[] states = section(); + + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + states[index(x, 7, z)] = SLAB; + } + } + states[index(8, 7, 8)] = LAMP; + SectionOpacity opacity = SectionOpacity.of(states, SOURCE); + + LightNibbles light = new LightPropagator().propagate(opacity); + + assertEquals(14, light.get(9, 7, 8), "the open side faces must let light through"); + assertEquals(14, light.get(8, 8, 8), "the top face is open as well"); + } + + @Test + void testTwoSourcesUseTheHigherLevel() { + int[] states = section(index(4, 8, 8), LAMP, index(12, 8, 8), LAMP); + SectionOpacity opacity = SectionOpacity.of(states, SOURCE); + + LightNibbles light = new LightPropagator().propagate(opacity); + + assertEquals(14, light.get(5, 8, 8)); + assertEquals(14, light.get(11, 8, 8)); + assertEquals(11, light.get(8, 8, 8), "the midpoint takes the brighter of both"); + } + + @Test + void testAFullyLitSectionCollapsesAgain() { + int[] states = new int[LightNibbles.BLOCK_COUNT]; + Arrays.fill(states, LAMP); + SectionOpacity opacity = SectionOpacity.of(states, SOURCE); + + LightNibbles light = new LightPropagator().propagate(opacity); + + assertTrue(light.isUniform(), "a section that is lit everywhere must not keep an array"); + assertEquals(15, light.get(3, 3, 3)); + } + + @Test + void testThePropagatorCanBeReusedWithoutBleedingResults() { + LightPropagator propagator = new LightPropagator(); + SectionOpacity lit = SectionOpacity.of(section(index(8, 8, 8), LAMP), SOURCE); + SectionOpacity dark = SectionOpacity.of(section(), SOURCE); + + LightNibbles first = propagator.propagate(lit); + LightNibbles second = propagator.propagate(dark); + + assertEquals(15, first.get(8, 8, 8), "the first result must stay untouched"); + assertEquals(0, second.get(8, 8, 8), "the reused buffers must not carry the previous run"); + } + + @Test + void testEachRunReturnsItsOwnResult() { + LightPropagator propagator = new LightPropagator(); + SectionOpacity opacity = SectionOpacity.of(section(index(8, 8, 8), LAMP), SOURCE); + + LightNibbles first = propagator.propagate(opacity); + LightNibbles second = propagator.propagate(opacity); + + assertNotNull(first); + assertNotNull(second); + assertEquals(first.get(8, 8, 8), second.get(8, 8, 8)); + } + + @Test + void testASourceAtTheBorderLightsInwards() { + SectionOpacity opacity = SectionOpacity.of(section(index(0, 0, 0), LAMP), SOURCE); + + LightNibbles light = new LightPropagator().propagate(opacity); + + assertEquals(15, light.get(0, 0, 0)); + assertEquals(14, light.get(1, 0, 0)); + assertEquals(14, light.get(0, 1, 0)); + } + + @Test + void testSourcesOfDifferentBrightnessDoNotOverflowTheQueue() { + // A position is queued again every time its level is raised. Dim sources are seeded first + // and spread low levels, then a bright source raises the very same positions, so a queue + // sized for "each position once" is too small. + BlockLightSource graded = new BlockLightSource() { + + @Override + public int emission(int stateId) { + return stateId == 0 ? 0 : stateId; + } + + @Override + public boolean blocksFace(int stateId, BlockFace face) { + return false; + } + }; + + int[] states = new int[LightNibbles.BLOCK_COUNT]; + + // Dim sources across the lower half, one bright source at the very top. + for (int y = 0; y < 8; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + states[index(x, y, z)] = 1 + ((x + y + z) % 3); + } + } + } + states[index(8, 15, 8)] = 15; + + LightNibbles light = new LightPropagator().propagate(SectionOpacity.of(states, graded)); + + assertEquals(15, light.get(8, 15, 8)); + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/light/MinestomBlockLightSourceTest.java b/src/test/java/net/theevilreaper/aves/instance/light/MinestomBlockLightSourceTest.java new file mode 100644 index 00000000..b5c88dae --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/light/MinestomBlockLightSourceTest.java @@ -0,0 +1,110 @@ +package net.theevilreaper.aves.instance.light; + +import net.minestom.server.instance.block.Block; +import net.minestom.testing.extension.MicrotusExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the adapter which answers the light properties of a block from the registry of the server. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +@ExtendWith(MicrotusExtension.class) +class MinestomBlockLightSourceTest { + + private final BlockLightSource source = new MinestomBlockLightSource(); + + @Test + void testAirBlocksNothing() { + for (BlockFace face : BlockFace.values()) { + assertFalse(this.source.blocksFace(Block.AIR.stateId(), face), "air must not block " + face); + } + } + + @Test + void testStoneBlocksEveryFace() { + for (BlockFace face : BlockFace.values()) { + assertTrue(this.source.blocksFace(Block.STONE.stateId(), face), "stone must block " + face); + } + } + + @Test + void testGlassBlocksNoFace() { + for (BlockFace face : BlockFace.values()) { + assertFalse(this.source.blocksFace(Block.GLASS.stateId(), face), "glass must not block " + face); + } + } + + @Test + void testGlowstoneReportsItsEmission() { + assertEquals(15, this.source.emission(Block.GLOWSTONE.stateId())); + } + + @Test + void testStoneEmitsNothing() { + assertEquals(0, this.source.emission(Block.STONE.stateId())); + } + + @Test + void testATorchReportsItsEmission() { + assertTrue(this.source.emission(Block.TORCH.stateId()) > 0, "a torch has to emit light"); + } + + @Test + void testABottomSlabBlocksItsBottomFaceOnly() { + // This is the case a single occlusion flag per block would answer wrongly. + int slab = Block.OAK_SLAB.withProperty("type", "bottom").stateId(); + + assertTrue(this.source.blocksFace(slab, BlockFace.BOTTOM)); + assertFalse(this.source.blocksFace(slab, BlockFace.TOP)); + } + + @Test + void testATopSlabBlocksItsTopFaceOnly() { + int slab = Block.OAK_SLAB.withProperty("type", "top").stateId(); + + assertTrue(this.source.blocksFace(slab, BlockFace.TOP)); + assertFalse(this.source.blocksFace(slab, BlockFace.BOTTOM)); + } + + @Test + void testAnUnknownStateIsTreatedAsTransparent() { + int unknown = Integer.MAX_VALUE; + + assertEquals(0, this.source.emission(unknown)); + assertFalse(this.source.blocksFace(unknown, BlockFace.TOP)); + } + + @Test + void testTheSourceFeedsAPropagationEndToEnd() { + int[] states = new int[LightNibbles.BLOCK_COUNT]; + states[(8 << 8) | (8 << 4) | 8] = Block.GLOWSTONE.stateId(); + + LightNibbles light = new LightPropagator().propagate(SectionOpacity.of(states, this.source)); + + assertEquals(15, light.get(8, 8, 8)); + assertEquals(14, light.get(9, 8, 8)); + } + + @Test + void testTheFaceOrderMatchesTheOneOfTheServer() { + // The adapter maps the faces by ordinal. If the server ever reorders its enum, every + // occlusion answer would silently refer to the wrong face. + net.minestom.server.instance.block.BlockFace[] serverFaces = + net.minestom.server.instance.block.BlockFace.values(); + + assertEquals(serverFaces.length, BlockFace.values().length); + + for (BlockFace face : BlockFace.values()) { + assertEquals(face.name(), serverFaces[face.ordinal()].name(), + "the face order of the engine and the server must stay identical"); + } + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/light/SectionOpacityTest.java b/src/test/java/net/theevilreaper/aves/instance/light/SectionOpacityTest.java new file mode 100644 index 00000000..34be0d68 --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/light/SectionOpacityTest.java @@ -0,0 +1,342 @@ +package net.theevilreaper.aves.instance.light; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the precomputed opacity table of a section. Resolving the properties of a block for every + * visited neighbour is the dominant cost of a light propagation, so they are looked up once per + * block and read from a table afterwards. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class SectionOpacityTest { + + private static final int AIR = 0; + private static final int STONE = 1; + private static final int GLOWSTONE = 2; + private static final int SLAB = 3; + + /** + * The first state id which no named block of this test uses. + * States built from it behave like air and only serve to fill a section with distinct ids. + */ + private static final int FILLER_BASE = 100; + + /** + * The amount of bytes a single table build may allocate. + *

+ * The two arrays a table keeps account for a little over eight kilobytes and the cache behind + * the build for a few more. The budget leaves room for both and still catches an allocation + * which happens per block rather than per section, because 4096 objects alone are far above it. + *

+ */ + private static final long ALLOCATION_BUDGET = 24L * 1024L; + + /** + * A source which describes four blocks without touching any registry. + * The slab blocks the downwards face only, which is what makes a single occlusion flag wrong. + */ + private static final BlockLightSource SOURCE = new BlockLightSource() { + + @Override + public int emission(int stateId) { + return stateId == GLOWSTONE ? 15 : 0; + } + + @Override + public boolean blocksFace(int stateId, BlockFace face) { + return switch (stateId) { + case STONE -> true; + case SLAB -> face == BlockFace.BOTTOM; + default -> false; + }; + } + }; + + /** + * Builds a section in which every block holds the given state id. + * + * @param stateId the state id of every block + * @return the created table + */ + private static SectionOpacity uniformSection(int stateId) { + int[] states = new int[LightNibbles.BLOCK_COUNT]; + java.util.Arrays.fill(states, stateId); + return SectionOpacity.of(states, SOURCE); + } + + @Test + void testATransparentBlockBlocksNoFace() { + SectionOpacity opacity = uniformSection(AIR); + + for (BlockFace face : BlockFace.values()) { + assertFalse(opacity.blocksFace(0, 0, 0, face), "air must not block " + face); + } + } + + @Test + void testAnOpaqueBlockBlocksEveryFace() { + SectionOpacity opacity = uniformSection(STONE); + + for (BlockFace face : BlockFace.values()) { + assertTrue(opacity.blocksFace(5, 5, 5, face), "stone must block " + face); + } + } + + @Test + void testADirectionalBlockOnlyBlocksItsOwnFaces() { + // Slabs, stairs, snow and farmland occlude some faces and not others. A table which stores + // a single flag per block would answer this wrongly for roughly one in seven block types. + SectionOpacity opacity = uniformSection(SLAB); + + assertTrue(opacity.blocksFace(1, 1, 1, BlockFace.BOTTOM)); + assertFalse(opacity.blocksFace(1, 1, 1, BlockFace.TOP)); + assertFalse(opacity.blocksFace(1, 1, 1, BlockFace.NORTH)); + } + + @Test + void testTheEmissionOfABlockIsKept() { + SectionOpacity opacity = uniformSection(GLOWSTONE); + + assertEquals(15, opacity.emission(3, 4, 5)); + } + + @Test + void testABlockWithoutEmissionReportsZero() { + assertEquals(0, uniformSection(STONE).emission(0, 0, 0)); + } + + @Test + void testASectionWithoutAnyEmissionIsReported() { + assertFalse(uniformSection(STONE).hasEmission()); + } + + @Test + void testASectionWithAnEmittingBlockIsReported() { + int[] states = new int[LightNibbles.BLOCK_COUNT]; + states[42] = GLOWSTONE; + + assertTrue(SectionOpacity.of(states, SOURCE).hasEmission()); + } + + @Test + void testAFullyTransparentSectionIsReported() { + assertTrue(uniformSection(AIR).isFullyTransparent()); + assertFalse(uniformSection(STONE).isFullyTransparent()); + } + + @Test + void testMixedBlocksAreResolvedPerPosition() { + int[] states = new int[LightNibbles.BLOCK_COUNT]; + states[index(2, 3, 4)] = STONE; + states[index(2, 3, 5)] = GLOWSTONE; + SectionOpacity opacity = SectionOpacity.of(states, SOURCE); + + assertTrue(opacity.blocksFace(2, 3, 4, BlockFace.TOP)); + assertFalse(opacity.blocksFace(2, 3, 5, BlockFace.TOP)); + assertEquals(15, opacity.emission(2, 3, 5)); + assertEquals(0, opacity.emission(2, 3, 4)); + } + + @Test + void testEveryDistinctStateIsResolvedOnlyOnce() { + int[] states = new int[LightNibbles.BLOCK_COUNT]; + java.util.Arrays.fill(states, STONE); + CountingSource counting = new CountingSource(); + + SectionOpacity.of(states, counting); + + assertEquals(1, counting.resolved, "4096 blocks of one state must cost one lookup"); + } + + @Test + void testTheStateArrayMustCoverTheWholeSection() { + assertThrows(IllegalArgumentException.class, () -> SectionOpacity.of(new int[10], SOURCE)); + } + + @Test + void testEveryDistinctStateOfAMixedSectionIsResolvedOnlyOnce() { + // The uniform shortcut never touches the cache, so a uniform section cannot show whether + // the table really resolves once per distinct state. Only a mixed section can. + int[] states = new int[LightNibbles.BLOCK_COUNT]; + + for (int index = 0; index < states.length; index++) { + states[index] = FILLER_BASE + (index % 200); + } + CountingSource counting = new CountingSource(); + + SectionOpacity.of(states, counting); + + assertEquals(200, counting.resolved, "200 distinct states must cost 200 lookups"); + } + + @Test + void testASectionOfNothingButDistinctStatesIsResolvedOnlyOnce() { + // A section may hold as many distinct states as it holds blocks. The cache behind the table + // has to reach that size without ever resolving a state twice. + int[] states = new int[LightNibbles.BLOCK_COUNT]; + + for (int index = 0; index < states.length; index++) { + states[index] = FILLER_BASE + index; + } + CountingSource counting = new CountingSource(); + + SectionOpacity.of(states, counting); + + assertEquals(LightNibbles.BLOCK_COUNT, counting.resolved); + } + + @Test + void testStatesWhichShareACacheSlotAreKeptApart() { + // State ids which are a multiple of the cache size apart land on the same slot of any cache + // that is indexed by a power of two. Confusing them would give a block the properties of an + // entirely different one. + int[] states = new int[LightNibbles.BLOCK_COUNT]; + int[] colliding = {STONE, STONE + 1024, STONE + 2048, GLOWSTONE, GLOWSTONE + 1024}; + + for (int index = 0; index < states.length; index++) { + states[index] = colliding[index % colliding.length]; + } + SectionOpacity opacity = SectionOpacity.of(states, SOURCE); + + for (int index = 0; index < states.length; index++) { + int x = index & 15; + int z = (index >> 4) & 15; + int y = index >> 8; + int stateId = states[index]; + + assertEquals(SOURCE.blocksFace(stateId, BlockFace.TOP), opacity.blocksFace(x, y, z, BlockFace.TOP)); + assertEquals(SOURCE.emission(stateId), opacity.emission(x, y, z)); + } + } + + @Test + void testTheTableIsBuiltWithoutGarbageBesidesItself() { + // The table used to resolve its states through a lambda that was created inside the loop + // over the blocks, which left one throwaway object per block behind. That is 4096 objects + // for a table that keeps two arrays of 4096 bytes, and it showed up as a light engine which + // scattered far more than the one of the server. The table must not allocate per block. + com.sun.management.ThreadMXBean threads = allocationCounter(); + int[] states = new int[LightNibbles.BLOCK_COUNT]; + + for (int index = 0; index < states.length; index++) { + states[index] = FILLER_BASE + (index % 200); + } + + // The very first build links the call sites of everything it touches, which allocates once + // and would be counted as if it belonged to the table. + SectionOpacity.of(states, SOURCE); + + long before = threads.getCurrentThreadAllocatedBytes(); + SectionOpacity built = SectionOpacity.of(states, SOURCE); + long allocated = threads.getCurrentThreadAllocatedBytes() - before; + + assertFalse(built.isUniform()); + assertTrue( + allocated <= ALLOCATION_BUDGET, + "building a table allocated " + allocated + " bytes, at most " + ALLOCATION_BUDGET + " are allowed" + ); + } + + /** + * Returns the bean which reports how many bytes the current thread has allocated. + * + * @return the bean of the running virtual machine + */ + private static com.sun.management.ThreadMXBean allocationCounter() { + java.lang.management.ThreadMXBean bean = java.lang.management.ManagementFactory.getThreadMXBean(); + + org.junit.jupiter.api.Assumptions.assumeTrue( + bean instanceof com.sun.management.ThreadMXBean, + "the running virtual machine does not report the allocation of a thread" + ); + return (com.sun.management.ThreadMXBean) bean; + } + + /** + * Calculates the index of a block inside a section. + * + * @param x the x coordinate inside the section + * @param y the y coordinate inside the section + * @param z the z coordinate inside the section + * @return the index of the block + */ + private static int index(int x, int y, int z) { + return (y << 8) | (z << 4) | x; + } + + /** + * A source which counts how often a distinct state was resolved. + */ + private static final class CountingSource implements BlockLightSource { + + private int resolved; + + @Override + public int emission(int stateId) { + this.resolved++; + return 0; + } + + @Override + public boolean blocksFace(int stateId, BlockFace face) { + return true; + } + } + + @Test + void testAUniformSectionIsRecognised() { + // Whole sections of a world hold one repeated state. Such a section needs no per position + // table at all, which saves both the lookups and the two arrays. + assertTrue(uniformSection(STONE).isUniform()); + assertTrue(uniformSection(AIR).isUniform()); + } + + @Test + void testASectionWithOneDifferingBlockIsNotUniform() { + int[] states = new int[LightNibbles.BLOCK_COUNT]; + states[LightNibbles.BLOCK_COUNT - 1] = STONE; + + assertFalse(SectionOpacity.of(states, SOURCE).isUniform()); + } + + @Test + void testAUniformSectionAnswersLikeAFullTable() { + SectionOpacity uniform = uniformSection(SLAB); + + // The shortcut must give the same answer at every position and for every face. + for (BlockFace face : BlockFace.values()) { + boolean expected = face == BlockFace.BOTTOM; + assertEquals(expected, uniform.blocksFace(0, 0, 0, face)); + assertEquals(expected, uniform.blocksFace(15, 15, 15, face)); + assertEquals(expected, uniform.blocksFace(7, 3, 11, face)); + } + assertEquals(0, uniform.emission(9, 9, 9)); + } + + @Test + void testAUniformSectionOfLampsStillReportsEmission() { + SectionOpacity uniform = uniformSection(GLOWSTONE); + + assertTrue(uniform.hasEmission()); + assertEquals(15, uniform.emission(4, 4, 4)); + } + + @Test + void testAUniformSectionResolvesExactlyOneState() { + int[] states = new int[LightNibbles.BLOCK_COUNT]; + java.util.Arrays.fill(states, STONE); + CountingSource counting = new CountingSource(); + + SectionOpacity.of(states, counting); + + assertEquals(1, counting.resolved); + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/light/SkyLightTest.java b/src/test/java/net/theevilreaper/aves/instance/light/SkyLightTest.java new file mode 100644 index 00000000..04cde0ba --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/light/SkyLightTest.java @@ -0,0 +1,189 @@ +package net.theevilreaper.aves.instance.light; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the sky light propagation. Sky light enters a chunk from above, falls straight down without + * losing a level until something stops it, and only then spreads like any other light. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class SkyLightTest { + + private static final int AIR = 0; + private static final int STONE = 1; + private static final int SLAB = 2; + + private static final BlockLightSource SOURCE = new BlockLightSource() { + + @Override + public int emission(int stateId) { + return 0; + } + + @Override + public boolean blocksFace(int stateId, BlockFace face) { + return switch (stateId) { + case STONE -> true; + case SLAB -> face == BlockFace.TOP; + default -> false; + }; + } + }; + + /** + * Builds the state ids of a chunk made of the given amount of air sections. + * + * @param sectionCount the amount of sections the chunk holds + * @return the state ids of every section + */ + private static List airChunk(int sectionCount) { + List sections = new ArrayList<>(sectionCount); + + for (int i = 0; i < sectionCount; i++) { + sections.add(new int[LightNibbles.BLOCK_COUNT]); + } + return sections; + } + + /** + * Converts the state ids of a chunk into opacity tables. + * + * @param sections the state ids of every section + * @return the opacity table of every section + */ + private static List tables(List sections) { + return sections.stream().map(states -> SectionOpacity.of(states, SOURCE)).toList(); + } + + /** + * Calculates the index of a block inside a section. + * + * @param x the x coordinate inside the section + * @param y the y coordinate inside the section + * @param z the z coordinate inside the section + * @return the index of the block + */ + private static int index(int x, int y, int z) { + return (y << 8) | (z << 4) | x; + } + + /** + * Fills a whole layer of the given section with a block. + * + * @param section the state ids of the section + * @param y the y coordinate inside the section + * @param stateId the state id to place + */ + private static void fillLayer(int[] section, int y, int stateId) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + section[index(x, y, z)] = stateId; + } + } + } + + @Test + void testAnOpenColumnIsFullyLit() { + List light = new ChunkLightPropagator().propagateSky(tables(airChunk(2))); + + assertEquals(15, light.get(1).get(8, 15, 8), "the top of the chunk sees the sky"); + assertEquals(15, light.get(1).get(8, 0, 8)); + assertEquals(15, light.get(0).get(8, 15, 8), "sky light does not weaken while falling"); + assertEquals(15, light.get(0).get(8, 0, 8), "it reaches the bottom of the chunk"); + } + + @Test + void testAnOpenChunkCollapsesToAUniformSection() { + List light = new ChunkLightPropagator().propagateSky(tables(airChunk(2))); + + for (LightNibbles section : light) { + assertTrue(section.isUniform(), "a fully lit section must not allocate an array"); + } + } + + @Test + void testAClosedCeilingStopsTheSkyLight() { + List sections = airChunk(2); + fillLayer(sections.get(1), 0, STONE); + + List light = new ChunkLightPropagator().propagateSky(tables(sections)); + + assertEquals(15, light.get(1).get(8, 1, 8), "above the ceiling stays lit"); + assertEquals(0, light.get(1).get(8, 0, 8), "the ceiling itself receives nothing"); + assertEquals(0, light.get(0).get(8, 15, 8), "everything below stays dark"); + } + + @Test + void testTheLightSpreadsSidewaysUnderAnOverhang() { + List sections = airChunk(2); + + // A ceiling covering everything but one column, so the sky light enters through the hole + // and then spreads horizontally, losing one level per block. + fillLayer(sections.get(1), 0, STONE); + sections.get(1)[index(8, 0, 8)] = AIR; + + List light = new ChunkLightPropagator().propagateSky(tables(sections)); + + assertEquals(15, light.get(1).get(8, 0, 8), "the open column keeps the full level"); + assertEquals(14, light.get(0).get(9, 15, 8), "the neighbour below the ceiling loses one"); + assertEquals(13, light.get(0).get(10, 15, 8)); + } + + @Test + void testADirectionalCeilingBlocksFromAbove() { + List sections = airChunk(2); + fillLayer(sections.get(1), 0, SLAB); + + List light = new ChunkLightPropagator().propagateSky(tables(sections)); + + assertEquals(0, light.get(1).get(8, 0, 8), "the slab blocks its top face"); + assertEquals(0, light.get(0).get(8, 15, 8)); + } + + @Test + void testEachColumnIsEvaluatedOnItsOwn() { + List sections = airChunk(1); + // A single pillar block in one column only. + sections.get(0)[index(4, 8, 4)] = STONE; + + List light = new ChunkLightPropagator().propagateSky(tables(sections)); + + assertEquals(15, light.get(0).get(4, 9, 4), "above the pillar stays lit"); + assertEquals(0, light.get(0).get(4, 8, 4), "the pillar itself is dark"); + assertEquals(15, light.get(0).get(5, 8, 4), "the neighbouring column is untouched"); + } + + @Test + void testTheLightBelowAPillarIsRestoredFromTheSide() { + List sections = airChunk(1); + sections.get(0)[index(4, 8, 4)] = STONE; + + List light = new ChunkLightPropagator().propagateSky(tables(sections)); + + // The column below the pillar is shadowed, but its neighbours are fully lit and feed it. + assertEquals(14, light.get(0).get(4, 7, 4)); + } + + @Test + void testSkyAndBlockLightAreCalculatedIndependently() { + List sections = airChunk(1); + fillLayer(sections.get(0), 15, STONE); + List tables = tables(sections); + ChunkLightPropagator propagator = new ChunkLightPropagator(); + + List sky = propagator.propagateSky(tables); + List block = propagator.propagate(tables); + + assertEquals(0, sky.get(0).get(8, 8, 8), "the closed ceiling keeps the sky light out"); + assertEquals(0, block.get(0).get(8, 8, 8), "there is no emitting block either"); + } +} diff --git a/src/test/java/net/theevilreaper/aves/instance/light/SkyLightUpdateTest.java b/src/test/java/net/theevilreaper/aves/instance/light/SkyLightUpdateTest.java new file mode 100644 index 00000000..000efbb3 --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/instance/light/SkyLightUpdateTest.java @@ -0,0 +1,242 @@ +package net.theevilreaper.aves.instance.light; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests the incremental sky light update of an already calculated chunk. + *

+ * Sky light has an origin no block holds: it falls in from above. An update therefore has to know + * how far down the sky reaches in every column, otherwise it cannot tell which positions lost their + * origin and which gained one. Both directions are covered here, together with the case in which + * the changed block is not the one that decides how far the sky reaches. + *

+ * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class SkyLightUpdateTest { + + private static final int AIR = 0; + private static final int STONE = 1; + + private static final int HEIGHT = 2 * LightNibbles.DIMENSION; + + private static final BlockLightSource SOURCE = new BlockLightSource() { + + @Override + public int emission(int stateId) { + return 0; + } + + @Override + public boolean blocksFace(int stateId, BlockFace face) { + return stateId == STONE; + } + }; + + /** + * Builds the state ids of a chunk made of the given amount of air sections. + * + * @param sectionCount the amount of sections the chunk holds + * @return the state ids of every section + */ + private static List airChunk(int sectionCount) { + List sections = new ArrayList<>(sectionCount); + + for (int i = 0; i < sectionCount; i++) { + sections.add(new int[LightNibbles.BLOCK_COUNT]); + } + return sections; + } + + /** + * Converts the state ids of a chunk into opacity tables. + * + * @param sections the state ids of every section + * @return the opacity table of every section + */ + private static List tables(List sections) { + return sections.stream().map(states -> SectionOpacity.of(states, SOURCE)).toList(); + } + + /** + * Calculates the index of a block inside a section. + * + * @param x the x coordinate inside the section + * @param y the y coordinate inside the section + * @param z the z coordinate inside the section + * @return the index of the block + */ + private static int index(int x, int y, int z) { + return (y << 8) | (z << 4) | x; + } + + /** + * Places a block at a position of the column, addressing the section it belongs to. + * + * @param sections the state ids of every section + * @param x the x coordinate inside the chunk + * @param y the y coordinate inside the column + * @param z the z coordinate inside the chunk + * @param stateId the state id to place + */ + private static void place(List sections, int x, int y, int z, int stateId) { + sections.get(y >> 4)[index(x, y & 15, z)] = stateId; + } + + /** + * Fills a whole layer of the column with a block. + * + * @param sections the state ids of every section + * @param y the y coordinate inside the column + * @param stateId the state id to place + */ + private static void fillLayer(List sections, int y, int stateId) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + place(sections, x, y, z, stateId); + } + } + } + + /** + * Asserts that the given state holds exactly what a calculation from scratch would produce. + * + * @param state the incrementally updated state + * @param sections the state ids of every section after the change + */ + private static void assertMatchesFullRecalculation(ChunkLightState state, List sections) { + ChunkLightState fresh = ChunkLightState.skyLight(tables(sections)); + + for (int y = 0; y < HEIGHT; y++) { + for (int z = 0; z < LightNibbles.DIMENSION; z++) { + for (int x = 0; x < LightNibbles.DIMENSION; x++) { + assertEquals(fresh.get(x, y, z), state.get(x, y, z), + "mismatch at " + x + "/" + y + "/" + z); + } + } + } + } + + @Test + void testPlacingABlockLowersTheSkyLightBelowIt() { + List sections = airChunk(2); + ChunkLightState state = ChunkLightState.skyLight(tables(sections)); + + place(sections, 8, 20, 8, STONE); + state.update(tables(sections), 8, 20, 8); + + assertEquals(0, state.get(8, 20, 8), "the new block itself carries no light"); + assertEquals(15, state.get(8, 21, 8), "above the block the sky is untouched"); + assertEquals(14, state.get(8, 19, 8), "below it the column only receives light from the side"); + } + + @Test + void testRemovingABlockingBlockLetsTheSkyFallThroughAgain() { + List sections = airChunk(2); + fillLayer(sections, 20, STONE); + ChunkLightState state = ChunkLightState.skyLight(tables(sections)); + + place(sections, 8, 20, 8, AIR); + state.update(tables(sections), 8, 20, 8); + + assertEquals(15, state.get(8, 20, 8), "the reopened position sees the sky again"); + assertEquals(15, state.get(8, 0, 8), "the sky falls down to the bottom of the column"); + assertEquals(14, state.get(9, 19, 8), "the neighbours below the ceiling are lit from the shaft"); + } + + @Test + void testTheIncrementalResultMatchesAFullRecalculationAfterAPlacement() { + List sections = airChunk(2); + ChunkLightState state = ChunkLightState.skyLight(tables(sections)); + + place(sections, 8, 20, 8, STONE); + state.update(tables(sections), 8, 20, 8); + + assertMatchesFullRecalculation(state, sections); + } + + @Test + void testTheIncrementalResultMatchesAFullRecalculationAfterARemoval() { + List sections = airChunk(2); + fillLayer(sections, 20, STONE); + ChunkLightState state = ChunkLightState.skyLight(tables(sections)); + + place(sections, 8, 20, 8, AIR); + state.update(tables(sections), 8, 20, 8); + + assertMatchesFullRecalculation(state, sections); + } + + @Test + void testAChangeBelowTheHighestBlockingBlockStaysCorrect() { + // A ceiling with a single shaft in it, so the space below is lit through that shaft. The + // change happens ten blocks below the ceiling and therefore leaves the reach of the sky in + // its column untouched, while the light around it still changes. + List sections = airChunk(2); + fillLayer(sections, 20, STONE); + place(sections, 7, 20, 8, AIR); + ChunkLightState state = ChunkLightState.skyLight(tables(sections)); + + place(sections, 8, 10, 8, STONE); + state.update(tables(sections), 8, 10, 8); + + assertEquals(0, state.get(8, 10, 8), "the new block itself carries no light"); + assertMatchesFullRecalculation(state, sections); + } + + @Test + void testRemovingTheHighestBlockUncoversTheOneBelowIt() { + List sections = airChunk(2); + fillLayer(sections, 20, STONE); + fillLayer(sections, 10, STONE); + ChunkLightState state = ChunkLightState.skyLight(tables(sections)); + + place(sections, 8, 20, 8, AIR); + state.update(tables(sections), 8, 20, 8); + + assertEquals(15, state.get(8, 11, 8), "the sky now reaches down to the next block"); + assertEquals(0, state.get(8, 10, 8), "the block below still stops it"); + assertMatchesFullRecalculation(state, sections); + } + + @Test + void testASequenceOfChangesStillMatchesAFullRecalculation() { + // A fixed seed keeps the sequence reproducible while still covering combinations no + // handwritten case would reach: changes above, below and at the height which stops the sky. + Random random = new Random(20_260_731L); + List sections = airChunk(2); + ChunkLightState state = ChunkLightState.skyLight(tables(sections)); + + for (int change = 0; change < 40; change++) { + int x = random.nextInt(LightNibbles.DIMENSION); + int y = random.nextInt(HEIGHT); + int z = random.nextInt(LightNibbles.DIMENSION); + + place(sections, x, y, z, random.nextBoolean() ? STONE : AIR); + state.update(tables(sections), x, y, z); + assertMatchesFullRecalculation(state, sections); + } + } + + @Test + void testRemovingABlockBelowTheHighestBlockingBlockStaysCorrect() { + List sections = airChunk(2); + fillLayer(sections, 20, STONE); + place(sections, 7, 20, 8, AIR); + place(sections, 8, 10, 8, STONE); + ChunkLightState state = ChunkLightState.skyLight(tables(sections)); + + place(sections, 8, 10, 8, AIR); + state.update(tables(sections), 8, 10, 8); + + assertMatchesFullRecalculation(state, sections); + } +} diff --git a/src/test/java/net/theevilreaper/aves/map/provider/ChunkLoaderFactoryTest.java b/src/test/java/net/theevilreaper/aves/map/provider/ChunkLoaderFactoryTest.java new file mode 100644 index 00000000..44372c59 --- /dev/null +++ b/src/test/java/net/theevilreaper/aves/map/provider/ChunkLoaderFactoryTest.java @@ -0,0 +1,66 @@ +package net.theevilreaper.aves.map.provider; + +import net.kyori.adventure.key.Key; +import net.minestom.server.instance.ChunkLoader; +import net.theevilreaper.aves.instance.anvil.AvesAnvilLoader; +import net.theevilreaper.aves.map.MapEntry; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Tests the factory which lets a map provider choose the chunk loader of an instance. + * + * @author TheMeinerLP + * @version 1.0.0 + * @since 1.16.0 + */ +class ChunkLoaderFactoryTest { + + private static final Key OVERWORLD = Key.key("minecraft:overworld"); + + @Test + void testTheAnvilFactoryCreatesTheAvesLoader(@TempDir Path tempDir) throws IOException { + Files.createDirectories(tempDir.resolve("world")); + MapEntry entry = MapEntry.of(tempDir.resolve("world")); + + ChunkLoader loader = ChunkLoaderFactory.anvil().create(entry, OVERWORLD); + + assertNotNull(loader); + assertInstanceOf(AvesAnvilLoader.class, loader); + ((AvesAnvilLoader) loader).close(); + } + + @Test + void testACustomFactoryIsUsedAsGiven(@TempDir Path tempDir) throws IOException { + Files.createDirectories(tempDir.resolve("world")); + MapEntry entry = MapEntry.of(tempDir.resolve("world")); + ChunkLoader expected = ChunkLoader.noop(); + + ChunkLoaderFactory factory = (mapEntry, dimension) -> expected; + + assertEquals(expected, factory.create(entry, OVERWORLD)); + } + + @Test + void testTheFactoryReceivesTheDirectoryOfTheEntry(@TempDir Path tempDir) throws IOException { + Path world = tempDir.resolve("world"); + Files.createDirectories(world); + MapEntry entry = MapEntry.of(world); + + ChunkLoaderFactory factory = (mapEntry, dimension) -> { + assertEquals(world, mapEntry.getDirectoryRoot()); + assertEquals(OVERWORLD, dimension); + return ChunkLoader.noop(); + }; + + assertNotNull(factory.create(entry, OVERWORLD)); + } +}