changes

Author: Palash Chauhan

README.md

@@ -77,12 +77,14 @@ Skip steps 1-2 above with the bundled Docker cluster. From a fresh clone:
 ```bash
 # 1. Bring up the full stack at the versions pinned in pom.xml and BLOCK
 #    until every container reports healthy (REST is ~30-60s on cold start).
-#    First time: ~8-12 min total; subsequent runs are cached.
+#    First time: ~8-12 min total -- most of that is Maven downloading
+#    ~1.5 GB of dependencies into the BuildKit cache mount. Subsequent
+#    runs reuse the cache and rebuild in seconds.
 docker compose -f docker/docker-compose.yml up -d --build --wait
 
 # 2. Validate it works end-to-end (CRUD + UpdateItem + BatchWriteItem + streams).
 bash docker/scripts/smoke.sh
-# -> "Result: 20 checks PASSED across 18 API calls"
+# -> "Result: 21 checks PASSED across 18 API calls"
 
 # 3. Use it. The DynamoDB-compatible endpoint is at http://localhost:8842 .
 #    Point any AWS SDK at it (Java/Python/Node.js snippets in

docker/Dockerfile.phoenix-adapters

@@ -34,8 +34,16 @@ RUN --mount=type=cache,target=/root/.m2 \
         clean package
 
 RUN set -eux; \
-    tarball=$(ls phoenix-ddb-assembly/target/phoenix-adapters-*-bin.tar.gz | head -n1); \
-    cp "${tarball}" /tmp/phoenix-adapters-bin.tar.gz
+    # If the assembly module ever ships an additional *-bin.tar.gz (e.g.
+    # with a classifier), fail loudly rather than silently picking one.
+    count=$(find phoenix-ddb-assembly/target -maxdepth 1 -type f -name 'phoenix-adapters-*-bin.tar.gz' | wc -l); \
+    if [ "$count" -ne 1 ]; then \
+        echo "Expected exactly one phoenix-adapters-*-bin.tar.gz, found $count:" >&2; \
+        find phoenix-ddb-assembly/target -maxdepth 1 -type f -name 'phoenix-adapters-*-bin.tar.gz' >&2; \
+        exit 1; \
+    fi; \
+    tarball=$(find phoenix-ddb-assembly/target -maxdepth 1 -type f -name 'phoenix-adapters-*-bin.tar.gz'); \
+    cp "$tarball" /tmp/phoenix-adapters-bin.tar.gz
 
 FROM eclipse-temurin:8-jdk-jammy
 

docker/README.md

@@ -48,13 +48,14 @@ From the **project root**:
 # 1. Bring up the full stack (ZK + HDFS + HBase+Phoenix + REST) and BLOCK
 #    until every service reports healthy (REST takes ~30-60s on a cold
 #    start because Phoenix has to bootstrap SYSTEM.* tables).
-#    First time: ~8-12 min (pulls upstream images + builds HBase/Phoenix + REST).
-#    Subsequent runs: cached.
+#    First time: ~8-12 min -- most of that is Maven downloading ~1.5 GB
+#    of dependencies into the BuildKit cache mount; subsequent runs reuse
+#    the cache and rebuild in seconds.
 docker compose -f docker/docker-compose.yml up -d --build --wait
 
 # 2. Validate it works end-to-end (CRUD + UpdateItem + BatchWriteItem + streams).
 bash docker/scripts/smoke.sh
-# -> "Result: 20 checks PASSED across 18 API calls"
+# -> "Result: 21 checks PASSED across 18 API calls"
 
 # 3. Use it. The DynamoDB-compatible REST endpoint is at http://localhost:8842 .
 #    Point any AWS SDK at it (Java/Python/Node.js snippets in
@@ -91,7 +92,7 @@ Inter-container traffic still uses the standard ports.
 ### Bring up just the cluster (no REST)
 
 ```bash
-docker compose -f docker/docker-compose.yml up -d --build \
+docker compose -f docker/docker-compose.yml up -d --build --wait \
     zookeeper namenode datanode hbase-master hbase-regionserver
 ```
 
@@ -224,6 +225,7 @@ HBase data persists across full `down`/`up` cycles.
 | Added a Maven dep to `phoenix-ddb-rest/pom.xml` | `--build phoenix-adapters-rest`. New dep downloads once; cache warms after. |
 | Clean slate | `docker compose ... down -v` then `up -d --build`. |
 | Code doesn't seem picked up | You ran `restart` instead of `up --build`. `restart` does not rebuild. |
+| Stack left running for days / many smoke iterations | HBase + REST logs grow unbounded inside the containers. `down -v` periodically to reclaim disk. |
 
 ### Pre-PR checklist
 
@@ -240,7 +242,7 @@ bash docker/scripts/smoke.sh
 docker compose -f docker/docker-compose.yml down -v
 ```
 
-If `smoke.sh` finishes with `Result: 20 checks PASSED across 18 API calls`,
+If `smoke.sh` finishes with `Result: 21 checks PASSED across 18 API calls`,
 your change is wire-compatible end to end through Phoenix on dockerized
 HBase across CRUD, batch, and the change-stream chain.
 
@@ -277,7 +279,7 @@ RPC controller:
 | `hbase.regionserver.wal.codec` | `…IndexedWALEditCodec` |
 | `hbase.region.server.rpc.scheduler.factory.class` | `…PhoenixRpcSchedulerFactory` |
 | `hbase.rpc.controllerfactory.class` | `…ServerRpcControllerFactory` |
-| `phoenix.task.handling.interval.ms` | `10` |
+| `phoenix.task.handling.interval.ms` | `1000` |
 | `phoenix.task.handling.initial.delay.ms` | `1` |
 
 `phoenix-server-hbase-2.5-5.3.1.jar` is copied into `${HBASE_HOME}/lib/` so

docker/conf/hbase/hbase-env.sh

@@ -9,6 +9,9 @@ export HBASE_HEAPSIZE=1G
 export HBASE_OFFHEAPSIZE=256m
 
 # Strip JDK11-specific GC flags HBase ships with; we run on JDK8.
+# This intentionally REPLACES the upstream value (rather than appending),
+# so any future upstream flag drops out of the container -- add new flags
+# to this list directly instead of re-deriving from upstream's HBASE_OPTS.
 export HBASE_OPTS="-XX:+UseG1GC -XX:+UnlockExperimentalVMOptions"
 export HBASE_MASTER_OPTS="${HBASE_OPTS} -Xms256m"
 export HBASE_REGIONSERVER_OPTS="${HBASE_OPTS} -Xms512m"

docker/conf/hbase/hbase-site.xml

@@ -17,6 +17,11 @@
         <name>hbase.zookeeper.property.clientPort</name>
         <value>2181</value>
     </property>
+    <!--
+        Dev-only: disables the HDFS hflush/hsync capability check so the
+        single-replica datanode in this stack can serve HBase WAL writes.
+        Do NOT copy this property into a production hbase-site.xml.
+    -->
     <property>
         <name>hbase.unsafe.stream.capability.enforce</name>
         <value>false</value>
@@ -47,9 +52,15 @@
         <name>hbase.rpc.controllerfactory.class</name>
         <value>org.apache.hadoop.hbase.ipc.controller.ServerRpcControllerFactory</value>
     </property>
+    <!--
+        Phoenix default is 60000 ms; we cut it to 1000 ms so dev-cluster
+        background tasks (index state transitions, dropped-table GC, etc.)
+        run quickly enough to keep iteration tight, without the laptop
+        CPU cost of a ~10 ms poll loop.
+    -->
     <property>
         <name>phoenix.task.handling.interval.ms</name>
-        <value>10</value>
+        <value>1000</value>
     </property>
     <property>
         <name>phoenix.task.handling.initial.delay.ms</name>

docker/conf/phoenix-adapters/hbase-site.xml

@@ -12,7 +12,7 @@
     </property>
     <property>
         <name>phoenix.task.handling.interval.ms</name>
-        <value>10</value>
+        <value>1000</value>
     </property>
     <property>
         <name>phoenix.task.handling.initial.delay.ms</name>

docker/docker-compose.yml

@@ -7,6 +7,11 @@
 
 name: phoenix-adapters
 
+# The container_name keys below pin each service to a fixed name (phx-*).
+# Combined with the fixed host ports (8842/9870/12181/etc.), this means
+# only one copy of the stack can run on a workstation at a time. That's
+# intentional given the host-port collisions; if you need parallel stacks,
+# drop the container_name keys AND change the host-port mappings.
 services:
 
   zookeeper:

docker/scripts/phoenix-adapters-entrypoint.sh

@@ -3,6 +3,26 @@ set -euo pipefail
 
 log() { echo "[phoenix-adapters][$(date -u +%H:%M:%S)] $*"; }
 
+# Guard against accidental reintroduction of the 3.4.x hadoop client jars.
+# Dockerfile.phoenix-adapters strips them because they reference
+# org.apache.hadoop.fs.WithErasureCoding (only present in hadoop-common
+# 3.4+), which poisons the client JVM via FileSystem ServiceLoader the
+# first time HBase returns a remote exception. If anyone re-adds them,
+# fail fast with a clear pointer instead of dying mid-bootstrap.
+shopt -s nullglob
+stray=( "${PHOENIX_ADAPTERS_HOME}/lib/hadoop-hdfs-"*.jar \
+        "${PHOENIX_ADAPTERS_HOME}/lib/hadoop-hdfs-client-"*.jar \
+        "${PHOENIX_ADAPTERS_HOME}/lib/hadoop-yarn-"*.jar \
+        "${PHOENIX_ADAPTERS_HOME}/lib/hadoop-mapreduce-client-"*.jar \
+        "${PHOENIX_ADAPTERS_HOME}/lib/hadoop-distcp-"*.jar )
+shopt -u nullglob
+if [[ ${#stray[@]} -gt 0 ]]; then
+    log "ERROR: assembly contains hadoop 3.4.x jars that must be stripped:"
+    for j in "${stray[@]}"; do log "  - ${j##*/}"; done
+    log "See the 'rm -f hadoop-hdfs-*' block in docker/Dockerfile.phoenix-adapters."
+    exit 1
+fi
+
 wait_for() {
     local host="$1" port="$2"
     log "Waiting for ${host}:${port} ..."

docker/scripts/smoke.sh

@@ -65,11 +65,22 @@ show_json() {
 LAST_RESP=""
 
 # Prints request + response visually and stashes the raw JSON in LAST_RESP.
+# Aborts immediately if the response is a DDB error envelope (has __type),
+# so per-step assertions don't have to translate confusing "expected X got
+# null" failures back into the underlying Phoenix error.
 ddb() {
     local action="$1" body="$2"
     show_json "request " "$body"
     LAST_RESP=$(curl -sS -X POST "$URL/" -H "$CT" -H "$TARGET.$action" -d "$body")
     show_json "response" "$LAST_RESP"
+    if printf '%s' "$LAST_RESP" | jq -e 'type == "object" and has("__type")' >/dev/null 2>&1; then
+        local err_type err_msg
+        err_type=$(printf '%s' "$LAST_RESP" | jq -r '.__type // "?"')
+        err_msg=$(printf  '%s' "$LAST_RESP" | jq -r '.Message // .message // ""')
+        printf "  ${RED}✗${RESET}  %s returned error ${B}%s${RESET}: %s\n" \
+            "$action" "$err_type" "$err_msg" >&2
+        exit 1
+    fi
 }
 
 assert_eq() {
@@ -97,7 +108,10 @@ assert_nonempty() {
 
 assert_ge() {
     local label="$1" actual="$2" threshold="$3"
-    if [[ "$actual" -ge "$threshold" ]]; then
+    # Coerce non-numeric (null, empty, "true", etc.) to 0 so the arithmetic
+    # comparison can't abort the script with "integer expression expected".
+    [[ "$actual" =~ ^-?[0-9]+$ ]] || actual=0
+    if (( actual >= threshold )); then
         printf "  ${GREEN}✓${RESET}  %s ${B}>=${RESET} %s (got %s)\n" "$label" "$threshold" "$actual"
         PASS=$((PASS + 1))
     else
@@ -280,7 +294,7 @@ iter=$(jq -r '.ShardIterator // empty' <<<"$LAST_RESP")
 assert_nonempty "ShardIterator" "$iter"
 
 step "GetRecords  (drain pages until empty)"
-total=0; pages=0; seen_keys=""
+total=0; pages=0; seen_keys=""; advanced=false
 while [[ -n "$iter" && "$iter" != "null" && $pages -lt 10 ]]; do
     pages=$((pages + 1))
     ddb GetRecords "{\"ShardIterator\":\"$iter\"}"
@@ -290,16 +304,27 @@ while [[ -n "$iter" && "$iter" != "null" && $pages -lt 10 ]]; do
     total=$((total + n))
     printf "  ${DIM}page %d: %d record(s)  keys=[%s]${RESET}\n" "$pages" "$n" "$keys"
     next=$(jq -r '.NextShardIterator // empty' <<<"$LAST_RESP")
-    if [[ "$next" == "$iter" || -z "$next" || "$next" == "null" ]]; then
+    # Iterator stuck-at-position guard: if NextShardIterator equals the
+    # current one across consecutive empty pages, the stream isn't actually
+    # being consumed, so further pages would just spin.
+    if [[ -z "$next" || "$next" == "null" || "$next" == "$iter" ]]; then
         break
     fi
+    advanced=true
     iter="$next"
     [[ $n -eq 0 ]] && break
 done
 printf "  ${DIM}total records: %d  keys=[%s]${RESET}\n" "$total" "$seen_keys"
 # Expect >= 4 mutations (PutItem-a, UpdateItem-a, PutItem-b, DeleteItem-b) plus
-# 3 from the batch (delete-a, put-c, put-d).
+# 3 from the batch (delete-a, put-c, put-d) -- 7 total in steady state.
 assert_ge "stream record count" "$total" "4"
+if $advanced; then
+    printf "  ${GREEN}✓${RESET}  ShardIterator advanced across pages\n"
+    PASS=$((PASS + 1))
+else
+    printf "  ${RED}✗${RESET}  ShardIterator never advanced; stream appears stuck\n" >&2
+    exit 1
+fi
 
 step "DeleteTable  (cleanup)"
 ddb DeleteTable "{\"TableName\":\"$TBL\"}"