From 2cea0550abfc16596dad9a8ee9599511c276c0ba Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Thu, 27 Aug 2026 17:55:22 +0300 Subject: [PATCH 01/20] IGNITE-27088 BinaryWriter should use internal String#value --- .../binary/JmhBinaryStringWriteBenchmark.java | 177 +++++++ .../internal/binary/BinaryWriterExImpl.java | 24 +- .../ignite/internal/binary/StringWriter.java | 454 ++++++++++++++++++ .../ignite/IgniteCommonsSystemProperties.java | 9 + .../direct/stream/DirectByteBufferStream.java | 9 +- .../internal/binary/StringWriterSelfTest.java | 208 ++++++++ .../streams/BinaryStreamsTestUtils.java | 8 + .../IgniteBinaryObjectsTestSuite.java | 2 + 8 files changed, 874 insertions(+), 17 deletions(-) create mode 100644 modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java create mode 100644 modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java new file mode 100644 index 0000000000000..6087226381c1d --- /dev/null +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.benchmarks.jmh.binary; + +import org.apache.ignite.internal.benchmarks.jmh.runner.JmhIdeBenchmarkRunner; +import org.apache.ignite.internal.binary.StringWriter; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.apache.ignite.internal.binary.streams.BinaryStreams; +import org.apache.ignite.internal.binary.streams.JmhBinaryStreamsFactory; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.profile.GCProfiler; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.ignite.IgniteCommonsSystemProperties.IGNITE_BINARY_STRING_ZERO_COPY; +import static org.openjdk.jmh.annotations.Mode.AverageTime; +import static org.openjdk.jmh.annotations.Scope.Thread; + +/** + * Compares zero-copy string serialization ({@link StringWriter} encoding directly into the stream buffer) with the + * legacy serialization through a temporary array produced by {@link String#getBytes}. The {@code zeroCopy} parameter + * selects the implementation via the {@code IGNITE_BINARY_STRING_ZERO_COPY} system property, which is set before the + * first use of {@link StringWriter} in each forked JVM. + */ +@State(Thread) +@OutputTimeUnit(NANOSECONDS) +@BenchmarkMode(AverageTime) +@Warmup(iterations = 5, time = 1, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = SECONDS) +public class JmhBinaryStringWriteBenchmark { + /** */ + @Param({"true", "false"}) + private boolean zeroCopy; + + /** */ + @Param({"8", "64", "512", "4096"}) + private int len; + + /** */ + @Param({"ascii", "latin1", "cyrillic", "mixed"}) + private String content; + + /** */ + @Param({"heap", "offheap"}) + private String stream; + + /** */ + private BinaryOutputStream out; + + /** */ + private String str; + + /** + * @param args Optional values of the {@code stream} parameter to run (e.g. {@code offheap}); all when empty. + */ + public static void main(String[] args) throws Exception { + OptionsBuilder builder = JmhIdeBenchmarkRunner.create() + .forks(1) + .benchmarks(JmhBinaryStringWriteBenchmark.class.getName()) + .profilers(GCProfiler.class) + .optionsBuilder(); + + if (args.length > 0) + builder.param("stream", args); + + new Runner(builder.build()).run(); + } + + /** */ + @Setup + public void setup() { + // Must be set before the first use of StringWriter in this JVM: the flag is read on class initialization. + System.setProperty(IGNITE_BINARY_STRING_ZERO_COPY, String.valueOf(zeroCopy)); + + StringBuilder sb = new StringBuilder(len); + + for (int i = 0; sb.length() < len; i++) { + switch (content) { + case "ascii": + sb.append((char)('a' + i % 26)); + + break; + + case "latin1": + // Every 8th char is a Latin-1 char with the sign bit set. + sb.append(i % 8 == 7 ? (char)(0xC0 + i % 0x20) : (char)('a' + i % 26)); + + break; + + case "cyrillic": + sb.append((char)('\u0410' + i % 32)); + + break; + + case "mixed": + // ASCII, Latin-1, 2-byte, 3-byte chars and a surrogate pair. + switch (i % 5) { + case 0: + sb.append((char)('a' + i % 26)); + + break; + + case 1: + sb.append('\u00e9'); + + break; + + case 2: + sb.append('\u0416'); + + break; + + case 3: + sb.append('\u20ac'); + + break; + + default: + sb.append("\ud83d\ude00"); + } + + break; + + default: + throw new IllegalArgumentException("Unknown content type: " + content); + } + } + + str = sb.toString(); + + out = "offheap".equals(stream) + ? JmhBinaryStreamsFactory.offheapOutputStream(4 * len + 64) + : BinaryStreams.outputStream(4 * len + 64); + } + + /** */ + @TearDown + public void tearDown() { + out.close(); + } + + /** */ + @Benchmark + public void writeString(Blackhole bh) { + out.position(0); + + StringWriter.write(str, out); + + bh.consume(out.position()); + } +} diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java index 033be969abcf7..0098bd38d8edd 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.UUID; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteCommonsSystemProperties; import org.apache.ignite.binary.BinaryObjectException; import org.apache.ignite.binary.BinaryRawWriter; import org.apache.ignite.internal.UnregisteredClassException; @@ -40,13 +41,16 @@ import org.apache.ignite.marshaller.Marshallers; import org.jetbrains.annotations.Nullable; -import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.ignite.IgniteCommonsSystemProperties.IGNITE_BINARY_STRING_ZERO_COPY; import static org.apache.ignite.internal.util.CommonUtils.MAX_ARRAY_SIZE; /** * Binary writer implementation. */ class BinaryWriterExImpl implements BinaryWriterEx { + /** Zero-copy serialization enabled flag. */ + static final boolean ZERO_COPY = IgniteCommonsSystemProperties.getBoolean(IGNITE_BINARY_STRING_ZERO_COPY, true); + /** Length: integer. */ private static final int LEN_INT = 4; @@ -733,20 +737,10 @@ void writeBooleanField(@Nullable Boolean val) { @Override public void writeString(@Nullable String val) throws BinaryObjectException { if (val == null) out.writeByte(GridBinaryMarshaller.NULL); - else { - byte[] strArr; - - if (BinaryUtils.USE_STR_SERIALIZATION_VER_2) - strArr = BinaryUtils.strToUtf8Bytes(val); - else - strArr = val.getBytes(UTF_8); - - out.unsafeEnsure(1 + 4); - out.unsafeWriteByte(GridBinaryMarshaller.STRING); - out.unsafeWriteInt(strArr.length); - - out.writeByteArray(strArr); - } + else if (ZERO_COPY) + StringWriter.write(val, out); + else + StringWriter.writeWithTemporaryArray(val, out); } /** {@inheritDoc} */ diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java new file mode 100644 index 0000000000000..91a1de2af0325 --- /dev/null +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java @@ -0,0 +1,454 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import org.apache.ignite.IgniteCommonsSystemProperties; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.apache.ignite.internal.util.GridUnsafe; +import org.jetbrains.annotations.NotNull; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.ignite.internal.binary.BinaryWriterExImpl.ZERO_COPY; + +/** + * Writes {@link String} values to a {@link BinaryOutputStream} in UTF-8 without allocation of temporary byte arrays: + * UTF-8 bytes are encoded directly into the stream buffer after a reserved length slot which is patched afterwards. + *

+ * On JDKs with compact strings (9+) an additional fast path is used: for Latin-1 strings the internal {@code byte[]} + * value of the string is encoded without per-char conversion, and for pure ASCII strings it is copied into the stream + * as-is, since the UTF-8 representation is identical to the internal one. + *

+ * The produced bytes are identical to serialization of {@code val.getBytes(UTF_8)}, including replacement of + * malformed surrogates with {@code '?'}. Zero-copy serialization can be disabled with the + * {@link IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY} system property. + */ +public final class StringWriter { + /** Latin-1 value of the {@code java.lang.String#coder} field. */ + private static final byte LATIN1 = 0; + + /** Mask to test 8 bytes for a set sign bit at once. */ + private static final long NEGATIVE_BYTES_MSK = 0x8080808080808080L; + + /** Offset of the {@code java.lang.String#value} field, or {@code -1} if the compact string fast path is unavailable. */ + private static final long STR_VALUE_OFF; + + /** Offset of the {@code java.lang.String#coder} field, or {@code -1} if the compact string fast path is unavailable. */ + private static final long STR_CODER_OFF; + + /** + * Handle of the intrinsified {@code java.lang.StringCoding#hasNegatives}, or {@code null} if unavailable. + * The intrinsic scans the array with SIMD instructions, far faster than any scalar loop. + */ + private static final MethodHandle HAS_NEGATIVES; + + static { + long valOff = -1; + long coderOff = -1; + + MethodHandle hasNegatives = null; + + if (ZERO_COPY) { + try { + Method mtd = Class.forName("java.lang.StringCoding") + .getDeclaredMethod("hasNegatives", byte[].class, int.class, int.class); + + // Requires '--add-opens=java.base/java.lang=ALL-UNNAMED' which Ignite scripts pass by default. + mtd.setAccessible(true); + + MethodHandle candidate = MethodHandles.lookup().unreflect(mtd); + + if (!(boolean)candidate.invokeExact(new byte[] {1, 2, 3}, 0, 3) + && (boolean)candidate.invokeExact(new byte[] {1, -2, 3}, 0, 3)) + hasNegatives = candidate; + } + catch (Throwable ignored) { + // The scalar implementation will be used. + } + } + + HAS_NEGATIVES = hasNegatives; + + if (ZERO_COPY) { + try { + Field valField = String.class.getDeclaredField("value"); + Field coderField = String.class.getDeclaredField("coder"); + + // On JDK 8 the value field is a char[], only the generic encoder can be used. + if (valField.getType() == byte[].class && coderField.getType() == byte.class) { + valOff = GridUnsafe.objectFieldOffset(valField); + coderOff = GridUnsafe.objectFieldOffset(coderField); + + if (!probe(valOff, coderOff)) { + valOff = -1; + coderOff = -1; + } + } + } + catch (Throwable ignored) { + valOff = -1; + coderOff = -1; + } + } + + STR_VALUE_OFF = valOff; + STR_CODER_OFF = coderOff; + } + + /** */ + private StringWriter() { + // No-op. + } + + /** + * Writes a string to the output stream as a {@link GridBinaryMarshaller#STRING} flag followed by UTF-8 length + * (int) and UTF-8 bytes. + * + * @param val Value. + * @param out Output stream. + */ + public static void write(@NotNull String val, BinaryOutputStream out) { + int len = val.length(); + + // Worst case is 3 bytes per char: a surrogate pair (2 chars) produces 4 bytes, a lone surrogate 1 byte. + // 1 byte for `GridBinaryMarshaller.STRING` and integer (4 bytes) for length. + int worstLen = 1 + 4 + 3 * len; + + if (worstLen + out.position() > Integer.MAX_VALUE) { + writeWithTemporaryArray(val, out); + + return; + } + + out.unsafeEnsure(worstLen); + + out.unsafeWriteByte(GridBinaryMarshaller.STRING); + + int lenPos = out.position(); + + out.unsafePosition(lenPos + 4); + + byte[] latin1 = latin1Value(val); + + int writtenBytes; + + if (out.hasArray()) { + // Encode into the backing array directly: plain indexed writes are much faster than + // per-byte virtual calls through the stream interface. + int start = lenPos + 4; + + int end = latin1 != null + ? encodeLatin1(latin1, out.array(), start) + : encodeChars(val, out.array(), start); + + writtenBytes = end - start; + + out.unsafePosition(end); + } + else { + writtenBytes = latin1 != null + ? writeLatin1(latin1, out) + : writeChars(val, out); + } + + out.unsafeWriteInt(lenPos, writtenBytes); + } + + /** + * Checks that the internal layout of {@link String} behaves as the compact string fast path expects. + * + * @param valOff Offset of the {@code value} field. + * @param coderOff Offset of the {@code coder} field. + * @return {@code True} if the fast path can be used. + */ + private static boolean probe(long valOff, long coderOff) { + String probe = "Ignite\u00e9"; + + // Compact strings can be disabled with -XX:-CompactStrings, then all strings are UTF-16 encoded. + if (GridUnsafe.getByteField(probe, coderOff) != LATIN1) + return false; + + Object val = GridUnsafe.getObjectField(probe, valOff); + + if (!(val instanceof byte[])) + return false; + + byte[] arr = (byte[])val; + + if (arr.length != probe.length()) + return false; + + for (int i = 0; i < arr.length; i++) { + if ((arr[i] & 0xFF) != probe.charAt(i)) + return false; + } + + return true; + } + + /** + * @param val String. + * @return Internal Latin-1 array of the string, or {@code null} if the string is UTF-16 encoded or the internal + * layout of {@link String} is unknown. + */ + public static byte[] latin1Value(String val) { + if (STR_VALUE_OFF < 0 || GridUnsafe.getByteField(val, STR_CODER_OFF) != LATIN1) + return null; + + return (byte[])GridUnsafe.getObjectField(val, STR_VALUE_OFF); + } + + /** + * Writes a Latin-1 encoded string value to the stream. Stream capacity must be ensured by the caller. + * + * @param val Internal Latin-1 array of the string. + * @param out Output stream. + * @return Number of bytes written. + */ + private static int writeLatin1(byte[] val, BinaryOutputStream out) { + if (!hasNegatives(val)) { + // Pure ASCII: UTF-8 representation matches the internal array, copy it as-is. + out.write(val, 0, val.length); + + return val.length; + } + + int utfLen = 0; + + for (int i = 0; i < val.length; i++) { + byte b = val[i]; + + if (b >= 0) { + out.unsafeWriteByte(b); + + utfLen++; + } + else { + int c = b & 0xFF; + + out.unsafeWriteByte((byte)(0xC0 | (c >> 6))); + out.unsafeWriteByte((byte)(0x80 | (c & 0x3F))); + + utfLen += 2; + } + } + + return utfLen; + } + + /** + * Writes string chars UTF-8 encoded to the stream. Replicates {@code String#getBytes(UTF_8)} behavior exactly, + * including replacement of malformed surrogates with {@code '?'}. Stream capacity must be ensured by the caller. + * + * @param val Value. + * @param out Output stream. + * @return Number of bytes written. + */ + private static int writeChars(String val, BinaryOutputStream out) { + int len = val.length(); + int utfLen = 0; + + for (int i = 0; i < len; i++) { + char c = val.charAt(i); + + if (c < 0x80) { + out.unsafeWriteByte((byte)c); + + utfLen++; + } + else if (c < 0x800) { + out.unsafeWriteByte((byte)(0xC0 | (c >> 6))); + out.unsafeWriteByte((byte)(0x80 | (c & 0x3F))); + + utfLen += 2; + } + else if (Character.isSurrogate(c)) { + char c2; + + if (Character.isHighSurrogate(c) && i + 1 < len && Character.isLowSurrogate(c2 = val.charAt(i + 1))) { + int cp = Character.toCodePoint(c, c2); + + out.unsafeWriteByte((byte)(0xF0 | (cp >> 18))); + out.unsafeWriteByte((byte)(0x80 | ((cp >> 12) & 0x3F))); + out.unsafeWriteByte((byte)(0x80 | ((cp >> 6) & 0x3F))); + out.unsafeWriteByte((byte)(0x80 | (cp & 0x3F))); + + utfLen += 4; + i++; + } + else { + out.unsafeWriteByte((byte)'?'); + + utfLen++; + } + } + else { + out.unsafeWriteByte((byte)(0xE0 | (c >> 12))); + out.unsafeWriteByte((byte)(0x80 | ((c >> 6) & 0x3F))); + out.unsafeWriteByte((byte)(0x80 | (c & 0x3F))); + + utfLen += 3; + } + } + + return utfLen; + } + + /** + * @param arr Array. + * @return {@code True} if the array contains a byte with the sign bit set. + */ + private static boolean hasNegatives(byte[] arr) { + if (HAS_NEGATIVES != null) { + try { + return (boolean)HAS_NEGATIVES.invokeExact(arr, 0, arr.length); + } + catch (Throwable ignored) { + // TODO LT log here. + // Fall through to the generic implementation. + } + } + + // 8-byte strides with an early exit: measured on par with a branch-free loop for clean arrays + // (the exit branch is never taken there, and neither variant is vectorized by the JIT) + // and far faster when a negative byte occurs early. + int i = 0; + + for (int lim = arr.length - Long.BYTES; i <= lim; i += Long.BYTES) { + if ((GridUnsafe.getLong(arr, GridUnsafe.BYTE_ARR_OFF + i) & NEGATIVE_BYTES_MSK) != 0) + return true; + } + + for (; i < arr.length; i++) { + if (arr[i] < 0) + return true; + } + + return false; + } + + /** + * Encodes a Latin-1 string value to the buffer as UTF-8. Buffer capacity must be ensured by the caller. + * + * @param val Internal Latin-1 array of the string. + * @param buf Buffer. + * @param pos Buffer position to encode to. + * @return Buffer position after the last encoded byte. + */ + private static int encodeLatin1(byte[] val, byte[] buf, int pos) { + if (!hasNegatives(val)) { + // Pure ASCII: UTF-8 representation matches the internal array, copy it as-is. + System.arraycopy(val, 0, buf, pos, val.length); + + return pos + val.length; + } + + // Unsafe writes skip the array bounds checks: capacity is ensured by the caller. + long off = GridUnsafe.BYTE_ARR_OFF + pos; + + for (int i = 0; i < val.length; i++) { + byte b = val[i]; + + if (b >= 0) + GridUnsafe.putByte(buf, off++, b); + else { + int c = b & 0xFF; + + GridUnsafe.putByte(buf, off++, (byte)(0xC0 | (c >> 6))); + GridUnsafe.putByte(buf, off++, (byte)(0x80 | (c & 0x3F))); + } + } + + return (int)(off - GridUnsafe.BYTE_ARR_OFF); + } + + /** + * Encodes string chars to the buffer as UTF-8. Replicates {@code String#getBytes(UTF_8)} behavior exactly, + * including replacement of malformed surrogates with {@code '?'}. Buffer capacity must be ensured by the caller. + * + * @param val Value. + * @param buf Buffer. + * @param pos Buffer position to encode to. + * @return Buffer position after the last encoded byte. + */ + private static int encodeChars(String val, byte[] buf, int pos) { + int len = val.length(); + + // Unsafe writes skip the array bounds checks: capacity is ensured by the caller. + long off = GridUnsafe.BYTE_ARR_OFF + pos; + + for (int i = 0; i < len; i++) { + char c = val.charAt(i); + + if (c < 0x80) + GridUnsafe.putByte(buf, off++, (byte)c); + else if (c < 0x800) { + GridUnsafe.putByte(buf, off++, (byte)(0xC0 | (c >> 6))); + GridUnsafe.putByte(buf, off++, (byte)(0x80 | (c & 0x3F))); + } + else if (Character.isSurrogate(c)) { + char c2; + + if (Character.isHighSurrogate(c) && i + 1 < len && Character.isLowSurrogate(c2 = val.charAt(i + 1))) { + int cp = Character.toCodePoint(c, c2); + + GridUnsafe.putByte(buf, off++, (byte)(0xF0 | (cp >> 18))); + GridUnsafe.putByte(buf, off++, (byte)(0x80 | ((cp >> 12) & 0x3F))); + GridUnsafe.putByte(buf, off++, (byte)(0x80 | ((cp >> 6) & 0x3F))); + GridUnsafe.putByte(buf, off++, (byte)(0x80 | (cp & 0x3F))); + + i++; + } + else + GridUnsafe.putByte(buf, off++, (byte)'?'); + } + else { + GridUnsafe.putByte(buf, off++, (byte)(0xE0 | (c >> 12))); + GridUnsafe.putByte(buf, off++, (byte)(0x80 | ((c >> 6) & 0x3F))); + GridUnsafe.putByte(buf, off++, (byte)(0x80 | (c & 0x3F))); + } + } + + return (int)(off - GridUnsafe.BYTE_ARR_OFF); + } + + /** + * Writes a string through a temporary UTF-8 byte array. + * + * @param val Value. + * @param out Output stream. + */ + static void writeWithTemporaryArray(String val, BinaryOutputStream out) { + byte[] strArr; + + if (BinaryUtils.USE_STR_SERIALIZATION_VER_2) + strArr = BinaryUtils.strToUtf8Bytes(val); + else + strArr = val.getBytes(UTF_8); + + // 1 byte for `GridBinaryMarshaller.STRING` and integer (4 bytes) for length. + out.unsafeEnsure(1 + 4); + out.unsafeWriteByte(GridBinaryMarshaller.STRING); + out.unsafeWriteInt(strArr.length); + + out.writeByteArray(strArr); + } +} diff --git a/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java b/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java index ec443398ce3e4..450e9d8636eb7 100644 --- a/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java +++ b/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java @@ -128,6 +128,15 @@ public class IgniteCommonsSystemProperties { public static final String IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2 = "IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2"; + /** + * When set to {@code false}, disables zero-copy UTF-8 serialization of {@link String} values by BinaryMarshaller, + * reverting to serialization through a temporary byte array produced by {@link String#getBytes}. + * Default value is {@code true}. + */ + @SystemProperty(value = "Enables zero-copy UTF-8 serialization of String values by BinaryMarshaller. " + + "When set to false, strings are serialized through a temporary byte array", defaults = "true") + public static final String IGNITE_BINARY_STRING_ZERO_COPY = "IGNITE_BINARY_STRING_ZERO_COPY"; + /** * Enables storage of typed arrays. * The default value is {@code BinaryUtils#DFLT_IGNITE_USE_BINARY_ARRAYS}. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java index c0752343b5441..21f418e8e373f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java @@ -33,6 +33,7 @@ import java.util.function.Supplier; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.binary.StringWriter; import org.apache.ignite.internal.managers.communication.CompressedMessage; import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheObject; @@ -733,8 +734,12 @@ public void writeBooleanArray(boolean[] val) { */ public void writeString(String val) { if (val != null) { - if (curStrBackingArr == null) - curStrBackingArr = val.getBytes(); + if (curStrBackingArr == null) { + curStrBackingArr = StringWriter.latin1Value(val); + + if (curStrBackingArr == null) + val.getBytes(); + } writeByteArray(curStrBackingArr); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java new file mode 100644 index 0000000000000..be203bbce1af5 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary; + +import java.util.Arrays; +import java.util.Random; +import org.apache.ignite.internal.binary.streams.BinaryOutputStream; +import org.apache.ignite.internal.binary.streams.BinaryStreams; +import org.apache.ignite.internal.binary.streams.BinaryStreamsTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Tests that {@link StringWriter} output is byte-identical to serialization of the {@link String#getBytes} result, + * which was used before zero-copy string serialization was introduced. + */ +public class StringWriterSelfTest extends GridCommonAbstractTest { + /** Edge cases for all encoder paths: ASCII bulk copy, Latin-1, generic UTF-16 and malformed surrogates. */ + private static final String[] CORPUS = { + "", + "a", + "?", + "abcdefghijklmnopqrstuvwxyz0123456789", // Long ASCII: exercises the 8-byte stride scan and bulk copy. + "caf\u00e9", // Latin-1 with a negative byte. + "\u00ff\u0080\u00a0", // Latin-1, negative bytes only. + "\u041f\u0440\u0438\u0432\u0435\u0442", // Cyrillic: 2-byte UTF-8 sequences. + "\u0800\u1234\uffff", // 3-byte UTF-8 sequences. + "\ud83d\ude00", // Emoji: valid surrogate pair. + "a\ud83d\ude00b\u00e9\u0416\u0001", // Mixed content. + "\ud800", // Lone high surrogate. + "\udc00", // Lone low surrogate. + "a\ud800", // High surrogate at the end. + "\ud800a", // High surrogate followed by a regular char. + "\ud800\ud800", // Two high surrogates. + "\udc00\ud800", // Low surrogate before a high one. + "\u0000", // NUL char. + "nul\u0000nul" + }; + + /** + * Tests corpus of edge case strings. + */ + @Test + public void testCorpus() { + for (String str : CORPUS) + check(str); + } + + /** + * Randomized differential test against {@link String#getBytes}. + */ + @Test + public void testRandomStrings() { + Random rnd = new Random(4242); + + for (int i = 0; i < 5_000; i++) { + int len = rnd.nextInt(65); + + StringBuilder sb = new StringBuilder(len); + + for (int j = 0; j < len; j++) { + int bucket = rnd.nextInt(100); + + char c; + + if (bucket < 40) + c = (char)rnd.nextInt(0x80); // ASCII. + else if (bucket < 55) + c = (char)(0x80 + rnd.nextInt(0x100 - 0x80)); // Latin-1. + else if (bucket < 65) + c = (char)(0x100 + rnd.nextInt(0x800 - 0x100)); // Other 2-byte chars. + else if (bucket < 75) + c = (char)(0x800 + rnd.nextInt(0xD800 - 0x800)); // 3-byte chars. + else if (bucket < 90) + c = (char)(0xD800 + rnd.nextInt(0xE000 - 0xD800)); // Surrogates, mostly malformed. + else + c = (char)(0xE000 + rnd.nextInt(0x10000 - 0xE000)); // 3-byte chars above the surrogate range. + + sb.append(c); + } + + check(sb.toString()); + } + } + + /** + * Tests that the stream position is correct after a string write, so surrounding values are not corrupted. + */ + @Test + public void testStreamPosition() { + // Small initial capacity to exercise buffer reallocation. + try (BinaryOutputStream out = BinaryStreams.outputStream(2)) { + out.writeInt(0xDEADBEEF); + + StringWriter.write("caf\u00e9", out); + StringWriter.write("\ud83d\ude00", out); + + out.writeInt(0xCAFEBABE); + + byte[] exp = concat( + intLE(0xDEADBEEF), + strBytes("caf\u00e9"), + strBytes("\ud83d\ude00"), + intLE(0xCAFEBABE)); + + assertTrue(Arrays.equals(exp, out.arrayCopy())); + } + } + + /** + * Checks that serialized form of the given string is byte-identical to serialization of + * the {@link String#getBytes} result. + * + * @param str String to check. + */ + private void check(String str) { + byte[] exp = strBytes(str); + + try (BinaryOutputStream out = BinaryStreams.outputStream(1)) { + StringWriter.write(str, out); + + assertSerialized(str, exp, out.arrayCopy()); + } + + try (BinaryOutputStream out = BinaryStreamsTestUtils.offheapOutputStream(1)) { + StringWriter.write(str, out); + + assertSerialized(str, exp, out.arrayCopy()); + } + } + + /** + * @param str Source string. + * @param exp Expected serialized form. + * @param act Actual serialized form. + */ + private void assertSerialized(String str, byte[] exp, byte[] act) { + if (!Arrays.equals(exp, act)) { + fail("String serialization mismatch [str=" + Arrays.toString(str.toCharArray()) + + ", exp=" + Arrays.toString(exp) + ", act=" + Arrays.toString(act) + ']'); + } + } + + /** + * @param str String. + * @return Expected serialized form of the string: flag, UTF-8 length and UTF-8 bytes. + */ + private static byte[] strBytes(String str) { + byte[] utf8 = str.getBytes(UTF_8); + + byte[] res = new byte[5 + utf8.length]; + + res[0] = GridBinaryMarshaller.STRING; + + System.arraycopy(intLE(utf8.length), 0, res, 1, 4); + System.arraycopy(utf8, 0, res, 5, utf8.length); + + return res; + } + + /** + * @param val Value. + * @return Little-endian representation of the value. + */ + private static byte[] intLE(int val) { + return new byte[] {(byte)val, (byte)(val >> 8), (byte)(val >> 16), (byte)(val >> 24)}; + } + + /** + * @param arrs Arrays. + * @return Concatenated arrays. + */ + private static byte[] concat(byte[]... arrs) { + int len = 0; + + for (byte[] arr : arrs) + len += arr.length; + + byte[] res = new byte[len]; + + int pos = 0; + + for (byte[] arr : arrs) { + System.arraycopy(arr, 0, res, pos, arr.length); + + pos += arr.length; + } + + return res; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/streams/BinaryStreamsTestUtils.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/streams/BinaryStreamsTestUtils.java index 4e6ae07254679..f8d16d30dacaa 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/streams/BinaryStreamsTestUtils.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/streams/BinaryStreamsTestUtils.java @@ -29,4 +29,12 @@ public class BinaryStreamsTestUtils { public static boolean threadLocalIsAcquired() { return THREAD_LOCAL.isAcquired(); } + + /** + * @param cap Initial capacity. + * @return Offheap output stream. + */ + public static BinaryOutputStream offheapOutputStream(int cap) { + return new BinaryOffheapOutputStream(cap); + } } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java index 69b639df6b1f9..bdc9230a585c8 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java @@ -46,6 +46,7 @@ import org.apache.ignite.internal.binary.GridDefaultBinaryMappersBinaryMetaDataSelfTest; import org.apache.ignite.internal.binary.GridSimpleLowerCaseBinaryMappersBinaryMetaDataSelfTest; import org.apache.ignite.internal.binary.RawBinaryObjectExtractorTest; +import org.apache.ignite.internal.binary.StringWriterSelfTest; import org.apache.ignite.internal.binary.builder.BinaryObjectBuilderAdditionalSelfTest; import org.apache.ignite.internal.binary.noncompact.BinaryFieldsHeapNonCompactSelfTest; import org.apache.ignite.internal.binary.noncompact.BinaryFieldsOffheapNonCompactSelfTest; @@ -107,6 +108,7 @@ BinaryTreeSelfTest.class, BinaryMarshallerSelfTest.class, + StringWriterSelfTest.class, BinaryObjectExceptionSelfTest.class, BinarySerialiedFieldComparatorSelfTest.class, From b6feacc14b6edec69d2eef4edba9927c49593b71 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Thu, 27 Aug 2026 18:01:45 +0300 Subject: [PATCH 02/20] IGNITE-27088 BinaryWriter should use internal String#value --- .../jmh/binary/JmhHasNegativesBenchmark.java | 143 ++++++++++++++++++ .../streams/JmhBinaryStreamsFactory.java | 31 ++++ 2 files changed, 174 insertions(+) create mode 100644 modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhHasNegativesBenchmark.java create mode 100644 modules/benchmarks/src/main/java/org/apache/ignite/internal/binary/streams/JmhBinaryStreamsFactory.java diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhHasNegativesBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhHasNegativesBenchmark.java new file mode 100644 index 0000000000000..3171f84915a05 --- /dev/null +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhHasNegativesBenchmark.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.benchmarks.jmh.binary; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.Method; +import java.util.Arrays; +import org.apache.ignite.internal.benchmarks.jmh.runner.JmhIdeBenchmarkRunner; +import org.apache.ignite.internal.util.GridUnsafe; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.openjdk.jmh.annotations.Mode.AverageTime; +import static org.openjdk.jmh.annotations.Scope.Thread; + +/** + * Compares implementations of a "byte array has a negative byte" scan: a branch-free 8-byte stride loop, + * the same loop with an early exit, and the intrinsified {@code java.lang.StringCoding#hasNegatives}. + */ +@State(Thread) +@OutputTimeUnit(NANOSECONDS) +@BenchmarkMode(AverageTime) +@Warmup(iterations = 3, time = 1, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 1, timeUnit = SECONDS) +public class JmhHasNegativesBenchmark { + /** Mask to test 8 bytes for a set sign bit at once. */ + private static final long NEGATIVE_BYTES_MSK = 0x8080808080808080L; + + /** Handle of the intrinsified {@code java.lang.StringCoding#hasNegatives}. */ + private static final MethodHandle HAS_NEGATIVES; + + static { + try { + Method mtd = Class.forName("java.lang.StringCoding") + .getDeclaredMethod("hasNegatives", byte[].class, int.class, int.class); + + mtd.setAccessible(true); + + HAS_NEGATIVES = MethodHandles.lookup().unreflect(mtd); + } + catch (Throwable e) { + throw new ExceptionInInitializerError(e); + } + } + + /** */ + @Param({"8", "64", "512", "4096"}) + private int len; + + /** */ + @Param({"clean", "dirtyStart", "dirtyEnd"}) + private String data; + + /** */ + private byte[] arr; + + /** */ + public static void main(String[] args) throws Exception { + JmhIdeBenchmarkRunner.create() + .forks(1) + .benchmarks(JmhHasNegativesBenchmark.class.getName()) + .run(); + } + + /** */ + @Setup + public void setup() { + arr = new byte[len]; + + Arrays.fill(arr, (byte)'a'); + + if ("dirtyStart".equals(data)) + arr[0] = -1; + else if ("dirtyEnd".equals(data)) + arr[len - 1] = -1; + } + + /** */ + @Benchmark + public boolean branchFree() { + long acc = 0; + + int i = 0; + + for (int lim = arr.length - Long.BYTES; i <= lim; i += Long.BYTES) + acc |= GridUnsafe.getLong(arr, GridUnsafe.BYTE_ARR_OFF + i); + + int tail = 0; + + for (; i < arr.length; i++) + tail |= arr[i]; + + return (acc & NEGATIVE_BYTES_MSK) != 0 || tail < 0; + } + + /** */ + @Benchmark + public boolean earlyExit() { + int i = 0; + + for (int lim = arr.length - Long.BYTES; i <= lim; i += Long.BYTES) { + if ((GridUnsafe.getLong(arr, GridUnsafe.BYTE_ARR_OFF + i) & NEGATIVE_BYTES_MSK) != 0) + return true; + } + + for (; i < arr.length; i++) { + if (arr[i] < 0) + return true; + } + + return false; + } + + /** */ + @Benchmark + public boolean intrinsic() throws Throwable { + return (boolean)HAS_NEGATIVES.invokeExact(arr, 0, arr.length); + } +} diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/binary/streams/JmhBinaryStreamsFactory.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/binary/streams/JmhBinaryStreamsFactory.java new file mode 100644 index 0000000000000..aefbdf00cf596 --- /dev/null +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/binary/streams/JmhBinaryStreamsFactory.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.binary.streams; + +/** + * Exposes package-private binary streams to benchmarks. + */ +public class JmhBinaryStreamsFactory { + /** + * @param cap Initial capacity. + * @return Offheap output stream. + */ + public static BinaryOutputStream offheapOutputStream(int cap) { + return new BinaryOffheapOutputStream(cap); + } +} From 9394099219c911cf849cc64b0b59ca34db5338ab Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 2 Sep 2026 13:38:19 +0300 Subject: [PATCH 03/20] IGNITE-27088 BinaryWriter should use internal String#value --- .../binary/JmhBinaryStringWriteBenchmark.java | 15 +- .../jmh/binary/JmhHasNegativesBenchmark.java | 143 ----------- .../streams/JmhBinaryStreamsFactory.java | 31 --- .../ignite/internal/binary/StringWriter.java | 182 +++++++------- .../internal/binary/StringWriterSelfTest.java | 224 +++++++++--------- 5 files changed, 216 insertions(+), 379 deletions(-) delete mode 100644 modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhHasNegativesBenchmark.java delete mode 100644 modules/benchmarks/src/main/java/org/apache/ignite/internal/binary/streams/JmhBinaryStreamsFactory.java diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java index 6087226381c1d..4dfb15f7118b9 100644 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java @@ -21,7 +21,6 @@ import org.apache.ignite.internal.binary.StringWriter; import org.apache.ignite.internal.binary.streams.BinaryOutputStream; import org.apache.ignite.internal.binary.streams.BinaryStreams; -import org.apache.ignite.internal.binary.streams.JmhBinaryStreamsFactory; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Measurement; @@ -44,15 +43,15 @@ /** * Compares zero-copy string serialization ({@link StringWriter} encoding directly into the stream buffer) with the - * legacy serialization through a temporary array produced by {@link String#getBytes}. The {@code zeroCopy} parameter + * legacy serialization through a temporary array produced by {@link String#getBytes()}. The {@code zeroCopy} parameter * selects the implementation via the {@code IGNITE_BINARY_STRING_ZERO_COPY} system property, which is set before the * first use of {@link StringWriter} in each forked JVM. */ @State(Thread) @OutputTimeUnit(NANOSECONDS) @BenchmarkMode(AverageTime) -@Warmup(iterations = 5, time = 1, timeUnit = SECONDS) -@Measurement(iterations = 5, time = 1, timeUnit = SECONDS) +@Warmup(iterations = 5, time = 5, timeUnit = SECONDS) +@Measurement(iterations = 5, time = 10, timeUnit = SECONDS) public class JmhBinaryStringWriteBenchmark { /** */ @Param({"true", "false"}) @@ -66,10 +65,6 @@ public class JmhBinaryStringWriteBenchmark { @Param({"ascii", "latin1", "cyrillic", "mixed"}) private String content; - /** */ - @Param({"heap", "offheap"}) - private String stream; - /** */ private BinaryOutputStream out; @@ -154,9 +149,7 @@ public void setup() { str = sb.toString(); - out = "offheap".equals(stream) - ? JmhBinaryStreamsFactory.offheapOutputStream(4 * len + 64) - : BinaryStreams.outputStream(4 * len + 64); + out = BinaryStreams.outputStream(4 * len + 64); } /** */ diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhHasNegativesBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhHasNegativesBenchmark.java deleted file mode 100644 index 3171f84915a05..0000000000000 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhHasNegativesBenchmark.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.benchmarks.jmh.binary; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.reflect.Method; -import java.util.Arrays; -import org.apache.ignite.internal.benchmarks.jmh.runner.JmhIdeBenchmarkRunner; -import org.apache.ignite.internal.util.GridUnsafe; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Warmup; - -import static java.util.concurrent.TimeUnit.NANOSECONDS; -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.openjdk.jmh.annotations.Mode.AverageTime; -import static org.openjdk.jmh.annotations.Scope.Thread; - -/** - * Compares implementations of a "byte array has a negative byte" scan: a branch-free 8-byte stride loop, - * the same loop with an early exit, and the intrinsified {@code java.lang.StringCoding#hasNegatives}. - */ -@State(Thread) -@OutputTimeUnit(NANOSECONDS) -@BenchmarkMode(AverageTime) -@Warmup(iterations = 3, time = 1, timeUnit = SECONDS) -@Measurement(iterations = 5, time = 1, timeUnit = SECONDS) -public class JmhHasNegativesBenchmark { - /** Mask to test 8 bytes for a set sign bit at once. */ - private static final long NEGATIVE_BYTES_MSK = 0x8080808080808080L; - - /** Handle of the intrinsified {@code java.lang.StringCoding#hasNegatives}. */ - private static final MethodHandle HAS_NEGATIVES; - - static { - try { - Method mtd = Class.forName("java.lang.StringCoding") - .getDeclaredMethod("hasNegatives", byte[].class, int.class, int.class); - - mtd.setAccessible(true); - - HAS_NEGATIVES = MethodHandles.lookup().unreflect(mtd); - } - catch (Throwable e) { - throw new ExceptionInInitializerError(e); - } - } - - /** */ - @Param({"8", "64", "512", "4096"}) - private int len; - - /** */ - @Param({"clean", "dirtyStart", "dirtyEnd"}) - private String data; - - /** */ - private byte[] arr; - - /** */ - public static void main(String[] args) throws Exception { - JmhIdeBenchmarkRunner.create() - .forks(1) - .benchmarks(JmhHasNegativesBenchmark.class.getName()) - .run(); - } - - /** */ - @Setup - public void setup() { - arr = new byte[len]; - - Arrays.fill(arr, (byte)'a'); - - if ("dirtyStart".equals(data)) - arr[0] = -1; - else if ("dirtyEnd".equals(data)) - arr[len - 1] = -1; - } - - /** */ - @Benchmark - public boolean branchFree() { - long acc = 0; - - int i = 0; - - for (int lim = arr.length - Long.BYTES; i <= lim; i += Long.BYTES) - acc |= GridUnsafe.getLong(arr, GridUnsafe.BYTE_ARR_OFF + i); - - int tail = 0; - - for (; i < arr.length; i++) - tail |= arr[i]; - - return (acc & NEGATIVE_BYTES_MSK) != 0 || tail < 0; - } - - /** */ - @Benchmark - public boolean earlyExit() { - int i = 0; - - for (int lim = arr.length - Long.BYTES; i <= lim; i += Long.BYTES) { - if ((GridUnsafe.getLong(arr, GridUnsafe.BYTE_ARR_OFF + i) & NEGATIVE_BYTES_MSK) != 0) - return true; - } - - for (; i < arr.length; i++) { - if (arr[i] < 0) - return true; - } - - return false; - } - - /** */ - @Benchmark - public boolean intrinsic() throws Throwable { - return (boolean)HAS_NEGATIVES.invokeExact(arr, 0, arr.length); - } -} diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/binary/streams/JmhBinaryStreamsFactory.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/binary/streams/JmhBinaryStreamsFactory.java deleted file mode 100644 index aefbdf00cf596..0000000000000 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/binary/streams/JmhBinaryStreamsFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.binary.streams; - -/** - * Exposes package-private binary streams to benchmarks. - */ -public class JmhBinaryStreamsFactory { - /** - * @param cap Initial capacity. - * @return Offheap output stream. - */ - public static BinaryOutputStream offheapOutputStream(int cap) { - return new BinaryOffheapOutputStream(cap); - } -} diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java index 91a1de2af0325..b3cd24b2c0ee7 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java @@ -46,7 +46,7 @@ public final class StringWriter { private static final byte LATIN1 = 0; /** Mask to test 8 bytes for a set sign bit at once. */ - private static final long NEGATIVE_BYTES_MSK = 0x8080808080808080L; + private static final long NEGATIVE_BYTES_MSK = 0b10000000_10000000_10000000_10000000_10000000_10000000_10000000_10000000L; /** Offset of the {@code java.lang.String#value} field, or {@code -1} if the compact string fast path is unavailable. */ private static final long STR_VALUE_OFF; @@ -61,24 +61,20 @@ public final class StringWriter { private static final MethodHandle HAS_NEGATIVES; static { - long valOff = -1; - long coderOff = -1; - MethodHandle hasNegatives = null; if (ZERO_COPY) { try { - Method mtd = Class.forName("java.lang.StringCoding") - .getDeclaredMethod("hasNegatives", byte[].class, int.class, int.class); + Method mtd = Class.forName("java.lang.StringCoding").getDeclaredMethod("hasNegatives", byte[].class, int.class, int.class); // Requires '--add-opens=java.base/java.lang=ALL-UNNAMED' which Ignite scripts pass by default. mtd.setAccessible(true); - MethodHandle candidate = MethodHandles.lookup().unreflect(mtd); + MethodHandle hasNegatives0 = MethodHandles.lookup().unreflect(mtd); - if (!(boolean)candidate.invokeExact(new byte[] {1, 2, 3}, 0, 3) - && (boolean)candidate.invokeExact(new byte[] {1, -2, 3}, 0, 3)) - hasNegatives = candidate; + if (!(boolean)hasNegatives0.invokeExact(new byte[] {1, 2, 3}, 0, 3) + && (boolean)hasNegatives0.invokeExact(new byte[] {1, -2, 3}, 0, 3)) + hasNegatives = hasNegatives0; } catch (Throwable ignored) { // The scalar implementation will be used. @@ -87,6 +83,9 @@ public final class StringWriter { HAS_NEGATIVES = hasNegatives; + long valOff = -1; + long coderOff = -1; + if (ZERO_COPY) { try { Field valField = String.class.getDeclaredField("value"); @@ -126,47 +125,41 @@ private StringWriter() { * @param out Output stream. */ public static void write(@NotNull String val, BinaryOutputStream out) { - int len = val.length(); - - // Worst case is 3 bytes per char: a surrogate pair (2 chars) produces 4 bytes, a lone surrogate 1 byte. - // 1 byte for `GridBinaryMarshaller.STRING` and integer (4 bytes) for length. - int worstLen = 1 + 4 + 3 * len; - - if (worstLen + out.position() > Integer.MAX_VALUE) { - writeWithTemporaryArray(val, out); - - return; - } - - out.unsafeEnsure(worstLen); - - out.unsafeWriteByte(GridBinaryMarshaller.STRING); - - int lenPos = out.position(); - - out.unsafePosition(lenPos + 4); - - byte[] latin1 = latin1Value(val); + int lenPos = writeHeader(out); int writtenBytes; - if (out.hasArray()) { - // Encode into the backing array directly: plain indexed writes are much faster than - // per-byte virtual calls through the stream interface. - int start = lenPos + 4; + byte[] latin1 = latin1Value(val); - int end = latin1 != null - ? encodeLatin1(latin1, out.array(), start) - : encodeChars(val, out.array(), start); + if (latin1 != null) { + if (out.hasArray()) { + // Encode into the backing array directly: plain indexed writes are much faster than + // per-byte virtual calls through the stream interface. + int start = lenPos + 4; + int end = encodeLatin1(latin1, out, start); - writtenBytes = end - start; + writtenBytes = end - start; - out.unsafePosition(end); + out.unsafePosition(end); + } + else + writtenBytes = writeLatin1(latin1, out); } else { - writtenBytes = latin1 != null - ? writeLatin1(latin1, out) - : writeChars(val, out); + // Worst case is 3 bytes per char: a surrogate pair (2 chars) produces 4 bytes, a lone surrogate 1 byte. + out.unsafeEnsure(Math.multiplyExact(3, val.length())); + + if (out.hasArray()) { + // Encode into the backing array directly: plain indexed writes are much faster than + // per-byte virtual calls through the stream interface. + int start = lenPos + 4; + int end = encodeChars(val, out.array(), start); + + writtenBytes = end - start; + + out.unsafePosition(end); + } else + writtenBytes = writeChars(val, out); } out.unsafeWriteInt(lenPos, writtenBytes); @@ -217,7 +210,40 @@ public static byte[] latin1Value(String val) { } /** - * Writes a Latin-1 encoded string value to the stream. Stream capacity must be ensured by the caller. + * @param arr Array. + * @return {@code True} if the array contains a byte with the sign bit set. + */ + private static boolean hasNegatives(byte[] arr) { + if (HAS_NEGATIVES != null) { + try { + return (boolean)HAS_NEGATIVES.invokeExact(arr, 0, arr.length); + } + catch (Throwable ignored) { + // TODO LT log here. + // Fall through to the generic implementation. + } + } + + // 8-byte strides with an early exit: measured on par with a branch-free loop for clean arrays + // (the exit branch is never taken there, and neither variant is vectorized by the JIT) + // and far faster when a negative byte occurs early. + int i = 0; + + for (int lim = arr.length - Long.BYTES; i <= lim; i += Long.BYTES) { + if ((GridUnsafe.getLong(arr, GridUnsafe.BYTE_ARR_OFF + i) & NEGATIVE_BYTES_MSK) != 0) + return true; + } + + for (; i < arr.length; i++) { + if (arr[i] < 0) + return true; + } + + return false; + } + + /** + * Writes a Latin-1 encoded string value to the stream. * * @param val Internal Latin-1 array of the string. * @param out Output stream. @@ -226,11 +252,13 @@ public static byte[] latin1Value(String val) { private static int writeLatin1(byte[] val, BinaryOutputStream out) { if (!hasNegatives(val)) { // Pure ASCII: UTF-8 representation matches the internal array, copy it as-is. - out.write(val, 0, val.length); + out.writeByteArray(val); return val.length; } + out.unsafeEnsure(Math.addExact(val.length, val.length)); + int utfLen = 0; for (int i = 0; i < val.length; i++) { @@ -313,55 +341,27 @@ else if (Character.isSurrogate(c)) { } /** - * @param arr Array. - * @return {@code True} if the array contains a byte with the sign bit set. - */ - private static boolean hasNegatives(byte[] arr) { - if (HAS_NEGATIVES != null) { - try { - return (boolean)HAS_NEGATIVES.invokeExact(arr, 0, arr.length); - } - catch (Throwable ignored) { - // TODO LT log here. - // Fall through to the generic implementation. - } - } - - // 8-byte strides with an early exit: measured on par with a branch-free loop for clean arrays - // (the exit branch is never taken there, and neither variant is vectorized by the JIT) - // and far faster when a negative byte occurs early. - int i = 0; - - for (int lim = arr.length - Long.BYTES; i <= lim; i += Long.BYTES) { - if ((GridUnsafe.getLong(arr, GridUnsafe.BYTE_ARR_OFF + i) & NEGATIVE_BYTES_MSK) != 0) - return true; - } - - for (; i < arr.length; i++) { - if (arr[i] < 0) - return true; - } - - return false; - } - - /** - * Encodes a Latin-1 string value to the buffer as UTF-8. Buffer capacity must be ensured by the caller. + * Encodes a Latin-1 string value to the buffer as UTF-8. * * @param val Internal Latin-1 array of the string. - * @param buf Buffer. + * @param out Output stream. * @param pos Buffer position to encode to. * @return Buffer position after the last encoded byte. */ - private static int encodeLatin1(byte[] val, byte[] buf, int pos) { + private static int encodeLatin1(byte[] val, BinaryOutputStream out, int pos) { if (!hasNegatives(val)) { + out.unsafeEnsure(val.length); + // Pure ASCII: UTF-8 representation matches the internal array, copy it as-is. - System.arraycopy(val, 0, buf, pos, val.length); + System.arraycopy(val, 0, out.array(), pos, val.length); return pos + val.length; } - // Unsafe writes skip the array bounds checks: capacity is ensured by the caller. + out.unsafeEnsure(Math.addExact(val.length, val.length)); + + byte[] buf = out.array(); + long off = GridUnsafe.BYTE_ARR_OFF + pos; for (int i = 0; i < val.length; i++) { @@ -436,7 +436,7 @@ else if (Character.isSurrogate(c)) { * @param val Value. * @param out Output stream. */ - static void writeWithTemporaryArray(String val, BinaryOutputStream out) { + static int writeWithTemporaryArray(String val, BinaryOutputStream out) { byte[] strArr; if (BinaryUtils.USE_STR_SERIALIZATION_VER_2) @@ -444,11 +444,21 @@ static void writeWithTemporaryArray(String val, BinaryOutputStream out) { else strArr = val.getBytes(UTF_8); + out.writeByteArray(strArr); + + return strArr.length; + } + + /** */ + private static int writeHeader(BinaryOutputStream out) { // 1 byte for `GridBinaryMarshaller.STRING` and integer (4 bytes) for length. out.unsafeEnsure(1 + 4); out.unsafeWriteByte(GridBinaryMarshaller.STRING); - out.unsafeWriteInt(strArr.length); - out.writeByteArray(strArr); + int pos = out.position(); + + out.unsafePosition(out.position() + 4); + + return pos; } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java index be203bbce1af5..2117ef3385361 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java @@ -18,7 +18,7 @@ package org.apache.ignite.internal.binary; import java.util.Arrays; -import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; import org.apache.ignite.internal.binary.streams.BinaryOutputStream; import org.apache.ignite.internal.binary.streams.BinaryStreams; import org.apache.ignite.internal.binary.streams.BinaryStreamsTestUtils; @@ -28,133 +28,165 @@ import static java.nio.charset.StandardCharsets.UTF_8; /** - * Tests that {@link StringWriter} output is byte-identical to serialization of the {@link String#getBytes} result, + * Tests that {@link StringWriter} output is byte-identical to serialization of the {@link String#getBytes()} result, * which was used before zero-copy string serialization was introduced. */ public class StringWriterSelfTest extends GridCommonAbstractTest { - /** Edge cases for all encoder paths: ASCII bulk copy, Latin-1, generic UTF-16 and malformed surrogates. */ - private static final String[] CORPUS = { - "", - "a", - "?", - "abcdefghijklmnopqrstuvwxyz0123456789", // Long ASCII: exercises the 8-byte stride scan and bulk copy. - "caf\u00e9", // Latin-1 with a negative byte. - "\u00ff\u0080\u00a0", // Latin-1, negative bytes only. - "\u041f\u0440\u0438\u0432\u0435\u0442", // Cyrillic: 2-byte UTF-8 sequences. - "\u0800\u1234\uffff", // 3-byte UTF-8 sequences. - "\ud83d\ude00", // Emoji: valid surrogate pair. - "a\ud83d\ude00b\u00e9\u0416\u0001", // Mixed content. - "\ud800", // Lone high surrogate. - "\udc00", // Lone low surrogate. - "a\ud800", // High surrogate at the end. - "\ud800a", // High surrogate followed by a regular char. - "\ud800\ud800", // Two high surrogates. - "\udc00\ud800", // Low surrogate before a high one. - "\u0000", // NUL char. - "nul\u0000nul" - }; + /** */ + public static final int ASCII_MAX = 0x80; - /** - * Tests corpus of edge case strings. - */ + /** */ + public static final int LATIN1_MAX = 0x100; + + /** */ + public static final int TWO_BYTES_MAX = 0x800; + + /** */ + public static final int THREE_BYTES_MAX = 0xD800; + + /** */ + public static final int FOUR_BYTES_MAX = 0xE000; + + /** */ + public static final int FOUR_BYTES_HIGH_BOUND = 0x10000; + + /** Tests for all encoder paths: ASCII bulk copy, Latin-1, generic UTF-16 and malformed surrogates. */ @Test public void testCorpus() { - for (String str : CORPUS) + String[] cases = { + "", + "a", + "?", + "abcdefghijklmnopqrstuvwxyz0123456789", // Long ASCII: exercises the 8-byte stride scan and bulk copy. + "caf\u00e9", // Latin-1 with a negative byte. + "\u00ff\u0080\u00a0", // Latin-1, negative bytes only. + "\u041f\u0440\u0438\u0432\u0435\u0442", // Cyrillic: 2-byte UTF-8 sequences. + "\u0800\u1234\uffff", // 3-byte UTF-8 sequences. + "\ud83d\ude00", // Emoji: valid surrogate pair. + "a\ud83d\ude00b\u00e9\u0416\u0001", // Mixed content. + "\ud800", // Lone high surrogate. + "\udc00", // Lone low surrogate. + "a\ud800", // High surrogate at the end. + "\ud800a", // High surrogate followed by a regular char. + "\ud800\ud800", // Two high surrogates. + "\udc00\ud800", // Low surrogate before a high one. + "\u0000", // NUL char. + "nul\u0000nul" + }; + + for (String str : cases) check(str); } - /** - * Randomized differential test against {@link String#getBytes}. - */ + /** Randomized differential test against {@link String#getBytes()}. */ @Test public void testRandomStrings() { - Random rnd = new Random(4242); - - for (int i = 0; i < 5_000; i++) { - int len = rnd.nextInt(65); + ThreadLocalRandom rnd = ThreadLocalRandom.current(); - StringBuilder sb = new StringBuilder(len); + for (int iter = 0; iter < 100; iter++) { + StringBuilder sb = new StringBuilder(1 + rnd.nextInt(42)); - for (int j = 0; j < len; j++) { + for (int i = 0; i < sb.capacity(); i++) { int bucket = rnd.nextInt(100); char c; if (bucket < 40) - c = (char)rnd.nextInt(0x80); // ASCII. + // ASCII. + c = (char)rnd.nextInt(ASCII_MAX); else if (bucket < 55) - c = (char)(0x80 + rnd.nextInt(0x100 - 0x80)); // Latin-1. + // Latin-1. + c = (char)(ASCII_MAX + rnd.nextInt(LATIN1_MAX - ASCII_MAX)); else if (bucket < 65) - c = (char)(0x100 + rnd.nextInt(0x800 - 0x100)); // Other 2-byte chars. + // Other 2-byte chars. + c = (char)(LATIN1_MAX + rnd.nextInt(TWO_BYTES_MAX - LATIN1_MAX)); else if (bucket < 75) - c = (char)(0x800 + rnd.nextInt(0xD800 - 0x800)); // 3-byte chars. + // 3-byte chars. + c = (char)(TWO_BYTES_MAX + rnd.nextInt(THREE_BYTES_MAX - TWO_BYTES_MAX)); else if (bucket < 90) - c = (char)(0xD800 + rnd.nextInt(0xE000 - 0xD800)); // Surrogates, mostly malformed. + // Surrogates, mostly malformed. + c = (char)(THREE_BYTES_MAX + rnd.nextInt(FOUR_BYTES_MAX - THREE_BYTES_MAX)); else - c = (char)(0xE000 + rnd.nextInt(0x10000 - 0xE000)); // 3-byte chars above the surrogate range. + // 3-byte chars above the surrogate range. + c = (char)(FOUR_BYTES_MAX + rnd.nextInt(FOUR_BYTES_HIGH_BOUND - FOUR_BYTES_MAX)); sb.append(c); } + assertFalse(sb.isEmpty()); + check(sb.toString()); } } /** - * Tests that the stream position is correct after a string write, so surrounding values are not corrupted. + * Tests strings whose UTF-8 form is larger than the stream's minimal capacity, so that the encoder's own capacity + * reservation (rather than the buffer's initial slack) is what keeps the unchecked writes in bounds. Covers every + * encoder path on both heap and offheap streams. */ @Test + public void testLargeStrings() { + int len = 100_000; + + StringBuilder ascii = new StringBuilder(len); + StringBuilder latin1 = new StringBuilder(len); + StringBuilder cyrillic = new StringBuilder(len); + StringBuilder mixed = new StringBuilder(len); + + for (int i = 0; i < len; i++) { + ascii.append((char)('a' + i % 26)); + // Every char is a Latin-1 char with the sign bit set: worst case for the 2-bytes-per-char reservation. + latin1.append((char)(ASCII_MAX + i % (LATIN1_MAX - ASCII_MAX))); + cyrillic.append((char)('\u0410' + i % 32)); + mixed.append((char)('a' + i % 26)).append('\u00e9').append('\u0416').append('\u20ac').append("\ud83d\ude00"); + } + + check(ascii.toString()); + check(latin1.toString()); + check(cyrillic.toString()); + check(mixed.toString()); + } + + /** Tests that the stream position is correct after a string write, so surrounding values are not corrupted. */ + @Test public void testStreamPosition() { - // Small initial capacity to exercise buffer reallocation. + int int1 = 0xDEADBEEF; + String str1 = "caf\u00e9"; + String str2 = "\ud83d\ude00"; + int int2 = 0xCAFEBABE; + + // Small initial capacity to check buffer reallocation. try (BinaryOutputStream out = BinaryStreams.outputStream(2)) { - out.writeInt(0xDEADBEEF); + out.writeInt(int1); + StringWriter.write(str1, out); + StringWriter.write(str2, out); + out.writeInt(int2); - StringWriter.write("caf\u00e9", out); - StringWriter.write("\ud83d\ude00", out); + byte[] strBytes1 = strBytes(str1); + byte[] strBytes2 = strBytes(str2); - out.writeInt(0xCAFEBABE); + byte[] exp = new byte[Integer.BYTES + strBytes1.length + strBytes2.length + Integer.BYTES]; - byte[] exp = concat( - intLE(0xDEADBEEF), - strBytes("caf\u00e9"), - strBytes("\ud83d\ude00"), - intLE(0xCAFEBABE)); + System.arraycopy(intBytes(int1), 0, exp, 0, Integer.BYTES); + System.arraycopy(strBytes1, 0, exp, Integer.BYTES, strBytes1.length); + System.arraycopy(strBytes2, 0, exp, Integer.BYTES + strBytes1.length, strBytes2.length); + System.arraycopy(intBytes(int2), 0, exp, Integer.BYTES + strBytes1.length + strBytes2.length, Integer.BYTES); assertTrue(Arrays.equals(exp, out.arrayCopy())); } } /** - * Checks that serialized form of the given string is byte-identical to serialization of - * the {@link String#getBytes} result. - * + * Checks that serialized form of the given string is byte-identical to serialization of the {@link String#getBytes()} result. * @param str String to check. */ private void check(String str) { - byte[] exp = strBytes(str); - - try (BinaryOutputStream out = BinaryStreams.outputStream(1)) { - StringWriter.write(str, out); - - assertSerialized(str, exp, out.arrayCopy()); - } + for (boolean heapStream : new boolean[] {true, false}) { + try (BinaryOutputStream out = heapStream ? BinaryStreams.outputStream(1) : BinaryStreamsTestUtils.offheapOutputStream(1)) { + StringWriter.write(str, out); - try (BinaryOutputStream out = BinaryStreamsTestUtils.offheapOutputStream(1)) { - StringWriter.write(str, out); - - assertSerialized(str, exp, out.arrayCopy()); - } - } - - /** - * @param str Source string. - * @param exp Expected serialized form. - * @param act Actual serialized form. - */ - private void assertSerialized(String str, byte[] exp, byte[] act) { - if (!Arrays.equals(exp, act)) { - fail("String serialization mismatch [str=" + Arrays.toString(str.toCharArray()) + - ", exp=" + Arrays.toString(exp) + ", act=" + Arrays.toString(act) + ']'); + assertTrue("String serialization mismatch: " + str, Arrays.equals(strBytes(str), out.arrayCopy())); + } } } @@ -163,14 +195,13 @@ private void assertSerialized(String str, byte[] exp, byte[] act) { * @return Expected serialized form of the string: flag, UTF-8 length and UTF-8 bytes. */ private static byte[] strBytes(String str) { - byte[] utf8 = str.getBytes(UTF_8); - - byte[] res = new byte[5 + utf8.length]; + byte[] bytes = str.getBytes(UTF_8); + byte[] res = new byte[Byte.BYTES + Integer.BYTES + bytes.length]; res[0] = GridBinaryMarshaller.STRING; - System.arraycopy(intLE(utf8.length), 0, res, 1, 4); - System.arraycopy(utf8, 0, res, 5, utf8.length); + System.arraycopy(intBytes(bytes.length), 0, res, 1, 4); + System.arraycopy(bytes, 0, res, 5, bytes.length); return res; } @@ -179,30 +210,7 @@ private static byte[] strBytes(String str) { * @param val Value. * @return Little-endian representation of the value. */ - private static byte[] intLE(int val) { + private static byte[] intBytes(int val) { return new byte[] {(byte)val, (byte)(val >> 8), (byte)(val >> 16), (byte)(val >> 24)}; } - - /** - * @param arrs Arrays. - * @return Concatenated arrays. - */ - private static byte[] concat(byte[]... arrs) { - int len = 0; - - for (byte[] arr : arrs) - len += arr.length; - - byte[] res = new byte[len]; - - int pos = 0; - - for (byte[] arr : arrs) { - System.arraycopy(arr, 0, res, pos, arr.length); - - pos += arr.length; - } - - return res; - } } From cbb0052dd3a4a08dcce1ba041c9a328d1e9a7f18 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 2 Sep 2026 14:16:33 +0300 Subject: [PATCH 04/20] IGNITE-27088 BinaryWriter should use internal String#value --- .../internal/binary/BinaryWriterExImpl.java | 17 +- .../ignite/internal/binary/StringWriter.java | 202 +++++++----------- .../ignite/IgniteCommonsSystemProperties.java | 3 +- 3 files changed, 98 insertions(+), 124 deletions(-) diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java index 0098bd38d8edd..12122b2acd01c 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java @@ -41,6 +41,7 @@ import org.apache.ignite.marshaller.Marshallers; import org.jetbrains.annotations.Nullable; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.ignite.IgniteCommonsSystemProperties.IGNITE_BINARY_STRING_ZERO_COPY; import static org.apache.ignite.internal.util.CommonUtils.MAX_ARRAY_SIZE; @@ -739,8 +740,20 @@ void writeBooleanField(@Nullable Boolean val) { out.writeByte(GridBinaryMarshaller.NULL); else if (ZERO_COPY) StringWriter.write(val, out); - else - StringWriter.writeWithTemporaryArray(val, out); + else { + byte[] strArr; + + if (BinaryUtils.USE_STR_SERIALIZATION_VER_2) + strArr = BinaryUtils.strToUtf8Bytes(val); + else + strArr = val.getBytes(UTF_8); + + out.unsafeEnsure(1 + 4); + out.unsafeWriteByte(GridBinaryMarshaller.STRING); + out.unsafeWriteInt(strArr.length); + + out.writeByteArray(strArr); + } } /** {@inheritDoc} */ diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java index b3cd24b2c0ee7..6bcdf9a656e19 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java @@ -24,22 +24,16 @@ import org.apache.ignite.IgniteCommonsSystemProperties; import org.apache.ignite.internal.binary.streams.BinaryOutputStream; import org.apache.ignite.internal.util.GridUnsafe; +import org.apache.ignite.lang.IgniteBiTuple; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.ignite.internal.binary.BinaryWriterExImpl.ZERO_COPY; /** - * Writes {@link String} values to a {@link BinaryOutputStream} in UTF-8 without allocation of temporary byte arrays: - * UTF-8 bytes are encoded directly into the stream buffer after a reserved length slot which is patched afterwards. - *

- * On JDKs with compact strings (9+) an additional fast path is used: for Latin-1 strings the internal {@code byte[]} - * value of the string is encoded without per-char conversion, and for pure ASCII strings it is copied into the stream - * as-is, since the UTF-8 representation is identical to the internal one. - *

- * The produced bytes are identical to serialization of {@code val.getBytes(UTF_8)}, including replacement of - * malformed surrogates with {@code '?'}. Zero-copy serialization can be disabled with the - * {@link IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY} system property. + * Writes {@link String} values to a {@link BinaryOutputStream} in UTF-8 without allocation of temporary byte arrays. + * + * @see IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY */ public final class StringWriter { /** Latin-1 value of the {@code java.lang.String#coder} field. */ @@ -54,63 +48,18 @@ public final class StringWriter { /** Offset of the {@code java.lang.String#coder} field, or {@code -1} if the compact string fast path is unavailable. */ private static final long STR_CODER_OFF; + static { + IgniteBiTuple result = fieldsOffsets(); + + STR_VALUE_OFF = result.get1(); + STR_CODER_OFF = result.get2(); + } + /** * Handle of the intrinsified {@code java.lang.StringCoding#hasNegatives}, or {@code null} if unavailable. * The intrinsic scans the array with SIMD instructions, far faster than any scalar loop. */ - private static final MethodHandle HAS_NEGATIVES; - - static { - MethodHandle hasNegatives = null; - - if (ZERO_COPY) { - try { - Method mtd = Class.forName("java.lang.StringCoding").getDeclaredMethod("hasNegatives", byte[].class, int.class, int.class); - - // Requires '--add-opens=java.base/java.lang=ALL-UNNAMED' which Ignite scripts pass by default. - mtd.setAccessible(true); - - MethodHandle hasNegatives0 = MethodHandles.lookup().unreflect(mtd); - - if (!(boolean)hasNegatives0.invokeExact(new byte[] {1, 2, 3}, 0, 3) - && (boolean)hasNegatives0.invokeExact(new byte[] {1, -2, 3}, 0, 3)) - hasNegatives = hasNegatives0; - } - catch (Throwable ignored) { - // The scalar implementation will be used. - } - } - - HAS_NEGATIVES = hasNegatives; - - long valOff = -1; - long coderOff = -1; - - if (ZERO_COPY) { - try { - Field valField = String.class.getDeclaredField("value"); - Field coderField = String.class.getDeclaredField("coder"); - - // On JDK 8 the value field is a char[], only the generic encoder can be used. - if (valField.getType() == byte[].class && coderField.getType() == byte.class) { - valOff = GridUnsafe.objectFieldOffset(valField); - coderOff = GridUnsafe.objectFieldOffset(coderField); - - if (!probe(valOff, coderOff)) { - valOff = -1; - coderOff = -1; - } - } - } - catch (Throwable ignored) { - valOff = -1; - coderOff = -1; - } - } - - STR_VALUE_OFF = valOff; - STR_CODER_OFF = coderOff; - } + private static final MethodHandle HAS_NEGATIVES = hasNegativesHandle(); /** */ private StringWriter() { @@ -125,7 +74,13 @@ private StringWriter() { * @param out Output stream. */ public static void write(@NotNull String val, BinaryOutputStream out) { - int lenPos = writeHeader(out); + // 1 byte for `GridBinaryMarshaller.STRING` and integer (4 bytes) for length. + out.unsafeEnsure(1 + 4); + out.unsafeWriteByte(GridBinaryMarshaller.STRING); + + int lenPos = out.position(); + + out.unsafePosition(out.position() + 4); int writtenBytes; @@ -158,45 +113,14 @@ public static void write(@NotNull String val, BinaryOutputStream out) { writtenBytes = end - start; out.unsafePosition(end); - } else + } + else writtenBytes = writeChars(val, out); } out.unsafeWriteInt(lenPos, writtenBytes); } - /** - * Checks that the internal layout of {@link String} behaves as the compact string fast path expects. - * - * @param valOff Offset of the {@code value} field. - * @param coderOff Offset of the {@code coder} field. - * @return {@code True} if the fast path can be used. - */ - private static boolean probe(long valOff, long coderOff) { - String probe = "Ignite\u00e9"; - - // Compact strings can be disabled with -XX:-CompactStrings, then all strings are UTF-16 encoded. - if (GridUnsafe.getByteField(probe, coderOff) != LATIN1) - return false; - - Object val = GridUnsafe.getObjectField(probe, valOff); - - if (!(val instanceof byte[])) - return false; - - byte[] arr = (byte[])val; - - if (arr.length != probe.length()) - return false; - - for (int i = 0; i < arr.length; i++) { - if ((arr[i] & 0xFF) != probe.charAt(i)) - return false; - } - - return true; - } - /** * @param val String. * @return Internal Latin-1 array of the string, or {@code null} if the string is UTF-16 encoded or the internal @@ -219,14 +143,11 @@ private static boolean hasNegatives(byte[] arr) { return (boolean)HAS_NEGATIVES.invokeExact(arr, 0, arr.length); } catch (Throwable ignored) { - // TODO LT log here. // Fall through to the generic implementation. } } - // 8-byte strides with an early exit: measured on par with a branch-free loop for clean arrays - // (the exit branch is never taken there, and neither variant is vectorized by the JIT) - // and far faster when a negative byte occurs early. + // 8-byte strides with an early exit. int i = 0; for (int lim = arr.length - Long.BYTES; i <= lim; i += Long.BYTES) { @@ -430,35 +351,76 @@ else if (Character.isSurrogate(c)) { return (int)(off - GridUnsafe.BYTE_ARR_OFF); } + /** */ + private static IgniteBiTuple fieldsOffsets() { + if (!ZERO_COPY) { + try { + Field valField = String.class.getDeclaredField("value"); + Field coderField = String.class.getDeclaredField("coder"); + + // On JDK 8 the value field is a char[], only the generic encoder can be used. + if (valField.getType() == byte[].class && coderField.getType() == byte.class) { + IgniteBiTuple res = new IgniteBiTuple<>( + GridUnsafe.objectFieldOffset(valField), + GridUnsafe.objectFieldOffset(coderField) + ); + + if(probe(res)) + return res; + } + } + catch (Throwable ignored) { + // No-op. + } + } + + return new IgniteBiTuple<>(-1L, -1L); + } + /** - * Writes a string through a temporary UTF-8 byte array. + * Checks that the internal layout of {@link String} behaves as the compact string fast path expects. * - * @param val Value. - * @param out Output stream. + * @param offsets Offsets of methods. + * @return {@code True} if the fast path can be used. */ - static int writeWithTemporaryArray(String val, BinaryOutputStream out) { - byte[] strArr; + private static boolean probe(IgniteBiTuple offsets) { + String probe = "Ignite\u00e9"; - if (BinaryUtils.USE_STR_SERIALIZATION_VER_2) - strArr = BinaryUtils.strToUtf8Bytes(val); - else - strArr = val.getBytes(UTF_8); + // Compact strings can be disabled with -XX:-CompactStrings, then all strings are UTF-16 encoded. + if (GridUnsafe.getByteField(probe, offsets.get1()) != LATIN1) + return false; - out.writeByteArray(strArr); + Object val = GridUnsafe.getObjectField(probe, offsets.get2()); + + if (!(val instanceof byte[] arr)) + return false; - return strArr.length; + if (arr.length != probe.length()) + return false; + + for (int i = 0; i < arr.length; i++) { + if ((arr[i] & 0xFF) != probe.charAt(i)) + return false; + } + + return true; } /** */ - private static int writeHeader(BinaryOutputStream out) { - // 1 byte for `GridBinaryMarshaller.STRING` and integer (4 bytes) for length. - out.unsafeEnsure(1 + 4); - out.unsafeWriteByte(GridBinaryMarshaller.STRING); + private static @Nullable MethodHandle hasNegativesHandle() { + if (ZERO_COPY) { + try { + Method mtd = Class.forName("java.lang.StringCoding").getDeclaredMethod("hasNegatives", byte[].class, int.class, int.class); - int pos = out.position(); + mtd.setAccessible(true); - out.unsafePosition(out.position() + 4); + return MethodHandles.lookup().unreflect(mtd); + } + catch (Throwable ignored) { + // No-op. + } + } - return pos; + return null; } } diff --git a/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java b/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java index 450e9d8636eb7..275ab07507a96 100644 --- a/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java +++ b/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java @@ -129,8 +129,7 @@ public class IgniteCommonsSystemProperties { "IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2"; /** - * When set to {@code false}, disables zero-copy UTF-8 serialization of {@link String} values by BinaryMarshaller, - * reverting to serialization through a temporary byte array produced by {@link String#getBytes}. + * When set to {@code false}, disables zero-copy UTF-8 serialization of {@link String} values by BinaryMarshaller. * Default value is {@code true}. */ @SystemProperty(value = "Enables zero-copy UTF-8 serialization of String values by BinaryMarshaller. " + From d36ef1a3df662b979a433f2168eeea278fce152f Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 4 Sep 2026 10:12:38 +0300 Subject: [PATCH 05/20] IGNITE-27088 BinaryWriter should use internal String#value --- .../binary/JmhBinaryStringWriteBenchmark.java | 15 +- .../ignite/internal/binary/StringWriter.java | 312 ++++++++---------- .../ignite/IgniteCommonsSystemProperties.java | 5 +- .../direct/stream/DirectByteBufferStream.java | 2 +- .../internal/binary/StringWriterSelfTest.java | 15 +- .../streams/BinaryStreamsTestUtils.java | 8 - 6 files changed, 141 insertions(+), 216 deletions(-) diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java index 4dfb15f7118b9..f029d50c8d76c 100644 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java @@ -42,10 +42,8 @@ import static org.openjdk.jmh.annotations.Scope.Thread; /** - * Compares zero-copy string serialization ({@link StringWriter} encoding directly into the stream buffer) with the - * legacy serialization through a temporary array produced by {@link String#getBytes()}. The {@code zeroCopy} parameter - * selects the implementation via the {@code IGNITE_BINARY_STRING_ZERO_COPY} system property, which is set before the - * first use of {@link StringWriter} in each forked JVM. + * Compares zero-copy string serialization with the legacy serialization. + * @see org.apache.ignite.IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY */ @State(Thread) @OutputTimeUnit(NANOSECONDS) @@ -71,9 +69,7 @@ public class JmhBinaryStringWriteBenchmark { /** */ private String str; - /** - * @param args Optional values of the {@code stream} parameter to run (e.g. {@code offheap}); all when empty. - */ + /** */ public static void main(String[] args) throws Exception { OptionsBuilder builder = JmhIdeBenchmarkRunner.create() .forks(1) @@ -81,16 +77,13 @@ public static void main(String[] args) throws Exception { .profilers(GCProfiler.class) .optionsBuilder(); - if (args.length > 0) - builder.param("stream", args); - new Runner(builder.build()).run(); } /** */ @Setup public void setup() { - // Must be set before the first use of StringWriter in this JVM: the flag is read on class initialization. + // Must be set before the first use of StringWriter in this JVM. System.setProperty(IGNITE_BINARY_STRING_ZERO_COPY, String.valueOf(zeroCopy)); StringBuilder sb = new StringBuilder(len); diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java index 6bcdf9a656e19..7615cf9af265e 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java @@ -39,9 +39,6 @@ public final class StringWriter { /** Latin-1 value of the {@code java.lang.String#coder} field. */ private static final byte LATIN1 = 0; - /** Mask to test 8 bytes for a set sign bit at once. */ - private static final long NEGATIVE_BYTES_MSK = 0b10000000_10000000_10000000_10000000_10000000_10000000_10000000_10000000L; - /** Offset of the {@code java.lang.String#value} field, or {@code -1} if the compact string fast path is unavailable. */ private static final long STR_VALUE_OFF; @@ -59,7 +56,7 @@ public final class StringWriter { * Handle of the intrinsified {@code java.lang.StringCoding#hasNegatives}, or {@code null} if unavailable. * The intrinsic scans the array with SIMD instructions, far faster than any scalar loop. */ - private static final MethodHandle HAS_NEGATIVES = hasNegativesHandle(); + private static final MethodHandle HAS_NEGATIVES = hasNegatives(); /** */ private StringWriter() { @@ -67,8 +64,7 @@ private StringWriter() { } /** - * Writes a string to the output stream as a {@link GridBinaryMarshaller#STRING} flag followed by UTF-8 length - * (int) and UTF-8 bytes. + * Writes a string to the output stream. * * @param val Value. * @param out Output stream. @@ -82,85 +78,41 @@ public static void write(@NotNull String val, BinaryOutputStream out) { out.unsafePosition(out.position() + 4); - int writtenBytes; + int written; byte[] latin1 = latin1Value(val); if (latin1 != null) { if (out.hasArray()) { - // Encode into the backing array directly: plain indexed writes are much faster than - // per-byte virtual calls through the stream interface. - int start = lenPos + 4; - int end = encodeLatin1(latin1, out, start); + if (!hasNegatives(latin1)) { + out.unsafeEnsure(latin1.length); + // Pure ASCII: UTF-8 representation matches the internal array, copy it as-is. + System.arraycopy(latin1, 0, out.array(), out.position(), latin1.length); - writtenBytes = end - start; + written = latin1.length; + } + else + written = encodeLatin1(latin1, out); - out.unsafePosition(end); + out.unsafePosition(out.position() + written); } else - writtenBytes = writeLatin1(latin1, out); + written = writeLatin1(latin1, out); } else { - // Worst case is 3 bytes per char: a surrogate pair (2 chars) produces 4 bytes, a lone surrogate 1 byte. + // Allocating memory for worst case - 3 bytes per char. out.unsafeEnsure(Math.multiplyExact(3, val.length())); if (out.hasArray()) { - // Encode into the backing array directly: plain indexed writes are much faster than - // per-byte virtual calls through the stream interface. - int start = lenPos + 4; - int end = encodeChars(val, out.array(), start); + written = encodeChars(val, out); - writtenBytes = end - start; - - out.unsafePosition(end); + out.unsafePosition(out.position() + written); } else - writtenBytes = writeChars(val, out); + written = writeChars(val, out); } - out.unsafeWriteInt(lenPos, writtenBytes); - } - - /** - * @param val String. - * @return Internal Latin-1 array of the string, or {@code null} if the string is UTF-16 encoded or the internal - * layout of {@link String} is unknown. - */ - public static byte[] latin1Value(String val) { - if (STR_VALUE_OFF < 0 || GridUnsafe.getByteField(val, STR_CODER_OFF) != LATIN1) - return null; - - return (byte[])GridUnsafe.getObjectField(val, STR_VALUE_OFF); - } - - /** - * @param arr Array. - * @return {@code True} if the array contains a byte with the sign bit set. - */ - private static boolean hasNegatives(byte[] arr) { - if (HAS_NEGATIVES != null) { - try { - return (boolean)HAS_NEGATIVES.invokeExact(arr, 0, arr.length); - } - catch (Throwable ignored) { - // Fall through to the generic implementation. - } - } - - // 8-byte strides with an early exit. - int i = 0; - - for (int lim = arr.length - Long.BYTES; i <= lim; i += Long.BYTES) { - if ((GridUnsafe.getLong(arr, GridUnsafe.BYTE_ARR_OFF + i) & NEGATIVE_BYTES_MSK) != 0) - return true; - } - - for (; i < arr.length; i++) { - if (arr[i] < 0) - return true; - } - - return false; + out.unsafeWriteInt(lenPos, written); } /** @@ -171,13 +123,6 @@ private static boolean hasNegatives(byte[] arr) { * @return Number of bytes written. */ private static int writeLatin1(byte[] val, BinaryOutputStream out) { - if (!hasNegatives(val)) { - // Pure ASCII: UTF-8 representation matches the internal array, copy it as-is. - out.writeByteArray(val); - - return val.length; - } - out.unsafeEnsure(Math.addExact(val.length, val.length)); int utfLen = 0; @@ -191,10 +136,10 @@ private static int writeLatin1(byte[] val, BinaryOutputStream out) { utfLen++; } else { - int c = b & 0xFF; + int c = b & 0b1111_1111; - out.unsafeWriteByte((byte)(0xC0 | (c >> 6))); - out.unsafeWriteByte((byte)(0x80 | (c & 0x3F))); + out.unsafeWriteByte((byte)(0b1100_0000 | (c >> 6))); + out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111))); utfLen += 2; } @@ -203,6 +148,36 @@ private static int writeLatin1(byte[] val, BinaryOutputStream out) { return utfLen; } + /** + * Encodes a Latin-1 string value to the buffer as UTF-8. + * + * @param val Internal Latin-1 array of the string. + * @param out Output stream. + * @return Count of written bytes. + */ + private static int encodeLatin1(byte[] val, BinaryOutputStream out) { + out.unsafeEnsure(Math.addExact(val.length, val.length)); + + byte[] buf = out.array(); + + long off = out.position() + GridUnsafe.BYTE_ARR_OFF; + + for (int i = 0; i < val.length; i++) { + byte b = val[i]; + + if (b >= 0) + GridUnsafe.putByte(buf, off++, b); + else { + int c = b & 0xFF; + + GridUnsafe.putByte(buf, off++, (byte)(0b1100_0000 | (c >> 6))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (c & 0b0011_1111))); + } + } + + return (int)(off - GridUnsafe.BYTE_ARR_OFF - out.position()); + } + /** * Writes string chars UTF-8 encoded to the stream. Replicates {@code String#getBytes(UTF_8)} behavior exactly, * including replacement of malformed surrogates with {@code '?'}. Stream capacity must be ensured by the caller. @@ -224,21 +199,28 @@ private static int writeChars(String val, BinaryOutputStream out) { utfLen++; } else if (c < 0x800) { - out.unsafeWriteByte((byte)(0xC0 | (c >> 6))); - out.unsafeWriteByte((byte)(0x80 | (c & 0x3F))); + out.unsafeWriteByte((byte)(0b11_000000 | (c >> 6))); + out.unsafeWriteByte((byte)(0b10_000000 | (c & 0b00_111111))); utfLen += 2; } - else if (Character.isSurrogate(c)) { + else if (!Character.isSurrogate(c)) { + out.unsafeWriteByte((byte)(0b1110_0000 | (c >> 12))); + out.unsafeWriteByte((byte)(0b1000_0000 | ((c >> 6) & 0b0011_1111))); + out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111))); + + utfLen += 3; + } + else { char c2; if (Character.isHighSurrogate(c) && i + 1 < len && Character.isLowSurrogate(c2 = val.charAt(i + 1))) { int cp = Character.toCodePoint(c, c2); - out.unsafeWriteByte((byte)(0xF0 | (cp >> 18))); - out.unsafeWriteByte((byte)(0x80 | ((cp >> 12) & 0x3F))); - out.unsafeWriteByte((byte)(0x80 | ((cp >> 6) & 0x3F))); - out.unsafeWriteByte((byte)(0x80 | (cp & 0x3F))); + out.unsafeWriteByte((byte)(0b1111_0000 | (cp >> 18))); + out.unsafeWriteByte((byte)(0b1000_0000 | ((cp >> 12) & 0b0011_1111))); + out.unsafeWriteByte((byte)(0b1000_0000 | ((cp >> 6) & 0b0011_1111))); + out.unsafeWriteByte((byte)(0b1000_0000 | (cp & 0b0011_1111))); utfLen += 4; i++; @@ -249,72 +231,25 @@ else if (Character.isSurrogate(c)) { utfLen++; } } - else { - out.unsafeWriteByte((byte)(0xE0 | (c >> 12))); - out.unsafeWriteByte((byte)(0x80 | ((c >> 6) & 0x3F))); - out.unsafeWriteByte((byte)(0x80 | (c & 0x3F))); - - utfLen += 3; - } } return utfLen; } - /** - * Encodes a Latin-1 string value to the buffer as UTF-8. - * - * @param val Internal Latin-1 array of the string. - * @param out Output stream. - * @param pos Buffer position to encode to. - * @return Buffer position after the last encoded byte. - */ - private static int encodeLatin1(byte[] val, BinaryOutputStream out, int pos) { - if (!hasNegatives(val)) { - out.unsafeEnsure(val.length); - - // Pure ASCII: UTF-8 representation matches the internal array, copy it as-is. - System.arraycopy(val, 0, out.array(), pos, val.length); - - return pos + val.length; - } - - out.unsafeEnsure(Math.addExact(val.length, val.length)); - - byte[] buf = out.array(); - - long off = GridUnsafe.BYTE_ARR_OFF + pos; - - for (int i = 0; i < val.length; i++) { - byte b = val[i]; - - if (b >= 0) - GridUnsafe.putByte(buf, off++, b); - else { - int c = b & 0xFF; - - GridUnsafe.putByte(buf, off++, (byte)(0xC0 | (c >> 6))); - GridUnsafe.putByte(buf, off++, (byte)(0x80 | (c & 0x3F))); - } - } - - return (int)(off - GridUnsafe.BYTE_ARR_OFF); - } - /** * Encodes string chars to the buffer as UTF-8. Replicates {@code String#getBytes(UTF_8)} behavior exactly, * including replacement of malformed surrogates with {@code '?'}. Buffer capacity must be ensured by the caller. * * @param val Value. - * @param buf Buffer. - * @param pos Buffer position to encode to. - * @return Buffer position after the last encoded byte. + * @param out Output stream. + * @return Count of written bytes. */ - private static int encodeChars(String val, byte[] buf, int pos) { + private static int encodeChars(String val, BinaryOutputStream out) { + byte[] buf = out.array(); int len = val.length(); // Unsafe writes skip the array bounds checks: capacity is ensured by the caller. - long off = GridUnsafe.BYTE_ARR_OFF + pos; + long off = GridUnsafe.BYTE_ARR_OFF + out.position(); for (int i = 0; i < len; i++) { char c = val.charAt(i); @@ -322,51 +257,93 @@ private static int encodeChars(String val, byte[] buf, int pos) { if (c < 0x80) GridUnsafe.putByte(buf, off++, (byte)c); else if (c < 0x800) { - GridUnsafe.putByte(buf, off++, (byte)(0xC0 | (c >> 6))); - GridUnsafe.putByte(buf, off++, (byte)(0x80 | (c & 0x3F))); + GridUnsafe.putByte(buf, off++, (byte)(0b1100_0000 | (c >> 6))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (c & 0b0011_1111))); } - else if (Character.isSurrogate(c)) { + else if (!Character.isSurrogate(c)) { + GridUnsafe.putByte(buf, off++, (byte)(0b1110_0000 | (c >> 12))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | ((c >> 6) & 0b0011_1111))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (c & 0b0011_1111))); + } + else { char c2; if (Character.isHighSurrogate(c) && i + 1 < len && Character.isLowSurrogate(c2 = val.charAt(i + 1))) { int cp = Character.toCodePoint(c, c2); - GridUnsafe.putByte(buf, off++, (byte)(0xF0 | (cp >> 18))); - GridUnsafe.putByte(buf, off++, (byte)(0x80 | ((cp >> 12) & 0x3F))); - GridUnsafe.putByte(buf, off++, (byte)(0x80 | ((cp >> 6) & 0x3F))); - GridUnsafe.putByte(buf, off++, (byte)(0x80 | (cp & 0x3F))); + GridUnsafe.putByte(buf, off++, (byte)(0b1111_0000 | (cp >> 18))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | ((cp >> 12) & 0b0011_1111))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | ((cp >> 6) & 0b0011_1111))); + GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (cp & 0b0011_1111))); i++; } else GridUnsafe.putByte(buf, off++, (byte)'?'); } - else { - GridUnsafe.putByte(buf, off++, (byte)(0xE0 | (c >> 12))); - GridUnsafe.putByte(buf, off++, (byte)(0x80 | ((c >> 6) & 0x3F))); - GridUnsafe.putByte(buf, off++, (byte)(0x80 | (c & 0x3F))); + } + + return (int)(off - GridUnsafe.BYTE_ARR_OFF - out.position()); + } + + /** + * @param val String. + * @return Internal Latin-1 array of the string, + * or {@code null} if the string is UTF-16 encoded or the internal layout of {@link String} is unknown. + */ + public static byte[] latin1Value(String val) { + if (STR_VALUE_OFF < 0 || GridUnsafe.getByteField(val, STR_CODER_OFF) != LATIN1) + return null; + + return (byte[])GridUnsafe.getObjectField(val, STR_VALUE_OFF); + } + + /** + * @param arr Array. + * @return {@code True} if the array contains a byte with the sign bit set. + */ + private static boolean hasNegatives(byte[] arr) { + if (HAS_NEGATIVES != null) { + try { + return (boolean)HAS_NEGATIVES.invokeExact(arr, 0, arr.length); } + catch (Throwable ignored) { + // Fall through to the generic implementation. + } + } + + // 8-byte strides with an early exit. + int i = 0; + + for (int lim = arr.length - Long.BYTES; i <= lim; i += Long.BYTES) { + long hasNegatives = GridUnsafe.getLong(arr, GridUnsafe.BYTE_ARR_OFF + i) + & 0b10000000_10000000_10000000_10000000_10000000_10000000_10000000_10000000L; + + if (hasNegatives != 0) + return true; } - return (int)(off - GridUnsafe.BYTE_ARR_OFF); + for (; i < arr.length; i++) { + if (arr[i] < 0) + return true; + } + + return false; } /** */ private static IgniteBiTuple fieldsOffsets() { - if (!ZERO_COPY) { + if (ZERO_COPY) { try { Field valField = String.class.getDeclaredField("value"); Field coderField = String.class.getDeclaredField("coder"); // On JDK 8 the value field is a char[], only the generic encoder can be used. if (valField.getType() == byte[].class && coderField.getType() == byte.class) { - IgniteBiTuple res = new IgniteBiTuple<>( + return new IgniteBiTuple<>( GridUnsafe.objectFieldOffset(valField), GridUnsafe.objectFieldOffset(coderField) ); - - if(probe(res)) - return res; } } catch (Throwable ignored) { @@ -377,37 +354,8 @@ private static IgniteBiTuple fieldsOffsets() { return new IgniteBiTuple<>(-1L, -1L); } - /** - * Checks that the internal layout of {@link String} behaves as the compact string fast path expects. - * - * @param offsets Offsets of methods. - * @return {@code True} if the fast path can be used. - */ - private static boolean probe(IgniteBiTuple offsets) { - String probe = "Ignite\u00e9"; - - // Compact strings can be disabled with -XX:-CompactStrings, then all strings are UTF-16 encoded. - if (GridUnsafe.getByteField(probe, offsets.get1()) != LATIN1) - return false; - - Object val = GridUnsafe.getObjectField(probe, offsets.get2()); - - if (!(val instanceof byte[] arr)) - return false; - - if (arr.length != probe.length()) - return false; - - for (int i = 0; i < arr.length; i++) { - if ((arr[i] & 0xFF) != probe.charAt(i)) - return false; - } - - return true; - } - /** */ - private static @Nullable MethodHandle hasNegativesHandle() { + private static @Nullable MethodHandle hasNegatives() { if (ZERO_COPY) { try { Method mtd = Class.forName("java.lang.StringCoding").getDeclaredMethod("hasNegatives", byte[].class, int.class, int.class); diff --git a/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java b/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java index 275ab07507a96..6bd0f7da69362 100644 --- a/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java +++ b/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java @@ -129,11 +129,10 @@ public class IgniteCommonsSystemProperties { "IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2"; /** - * When set to {@code false}, disables zero-copy UTF-8 serialization of {@link String} values by BinaryMarshaller. + * Enables zero-copy UTF-8 serialization of {@link String} values. * Default value is {@code true}. */ - @SystemProperty(value = "Enables zero-copy UTF-8 serialization of String values by BinaryMarshaller. " + - "When set to false, strings are serialized through a temporary byte array", defaults = "true") + @SystemProperty(value = "Enables zero-copy UTF-8 serialization of String values", defaults = "true") public static final String IGNITE_BINARY_STRING_ZERO_COPY = "IGNITE_BINARY_STRING_ZERO_COPY"; /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java index 21f418e8e373f..419e7835377fe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java @@ -738,7 +738,7 @@ public void writeString(String val) { curStrBackingArr = StringWriter.latin1Value(val); if (curStrBackingArr == null) - val.getBytes(); + curStrBackingArr = val.getBytes(); } writeByteArray(curStrBackingArr); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java index 2117ef3385361..c17219d51fd94 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java @@ -21,7 +21,6 @@ import java.util.concurrent.ThreadLocalRandom; import org.apache.ignite.internal.binary.streams.BinaryOutputStream; import org.apache.ignite.internal.binary.streams.BinaryStreams; -import org.apache.ignite.internal.binary.streams.BinaryStreamsTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; @@ -119,11 +118,7 @@ else if (bucket < 90) } } - /** - * Tests strings whose UTF-8 form is larger than the stream's minimal capacity, so that the encoder's own capacity - * reservation (rather than the buffer's initial slack) is what keeps the unchecked writes in bounds. Covers every - * encoder path on both heap and offheap streams. - */ + /** Tests strings whose UTF-8 form is larger than the stream's minimal capacity. */ @Test public void testLargeStrings() { int len = 100_000; @@ -181,12 +176,10 @@ public void testStreamPosition() { * @param str String to check. */ private void check(String str) { - for (boolean heapStream : new boolean[] {true, false}) { - try (BinaryOutputStream out = heapStream ? BinaryStreams.outputStream(1) : BinaryStreamsTestUtils.offheapOutputStream(1)) { - StringWriter.write(str, out); + try (BinaryOutputStream out = BinaryStreams.outputStream(1)) { + StringWriter.write(str, out); - assertTrue("String serialization mismatch: " + str, Arrays.equals(strBytes(str), out.arrayCopy())); - } + assertTrue("String serialization mismatch: " + str, Arrays.equals(strBytes(str), out.arrayCopy())); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/streams/BinaryStreamsTestUtils.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/streams/BinaryStreamsTestUtils.java index f8d16d30dacaa..4e6ae07254679 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/streams/BinaryStreamsTestUtils.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/streams/BinaryStreamsTestUtils.java @@ -29,12 +29,4 @@ public class BinaryStreamsTestUtils { public static boolean threadLocalIsAcquired() { return THREAD_LOCAL.isAcquired(); } - - /** - * @param cap Initial capacity. - * @return Offheap output stream. - */ - public static BinaryOutputStream offheapOutputStream(int cap) { - return new BinaryOffheapOutputStream(cap); - } } From 55feffdeb742fd681f6b489c399c21f0bd332d8d Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 4 Sep 2026 16:55:29 +0300 Subject: [PATCH 06/20] IGNITE-27088 BinaryWriter should use internal String#value --- .../test/java/org/apache/ignite/testsuites/ScriptTestSuite.java | 2 +- .../java/org/apache/ignite/IgniteCommonsSystemProperties.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/ScriptTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/ScriptTestSuite.java index fbed0087a3aeb..6c660b8097047 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/testsuites/ScriptTestSuite.java +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/ScriptTestSuite.java @@ -72,6 +72,6 @@ * */ @RunWith(ScriptTestRunner.class) -@ScriptRunnerTestsEnvironment(scriptsRoot = "modules/calcite/src/test/sql", timeout = 180000) +@ScriptRunnerTestsEnvironment(scriptsRoot = "modules/calcite/src/test/sql", timeout = 180000, regex = "test_replace") public class ScriptTestSuite { } diff --git a/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java b/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java index 6bd0f7da69362..ab43cde019ec2 100644 --- a/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java +++ b/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java @@ -132,7 +132,7 @@ public class IgniteCommonsSystemProperties { * Enables zero-copy UTF-8 serialization of {@link String} values. * Default value is {@code true}. */ - @SystemProperty(value = "Enables zero-copy UTF-8 serialization of String values", defaults = "true") + @SystemProperty(value = "Enables zero-copy UTF-8 serialization of String values", defaults = "false") public static final String IGNITE_BINARY_STRING_ZERO_COPY = "IGNITE_BINARY_STRING_ZERO_COPY"; /** From ccb6f145a13273b450ea3d775fe4d774c75ca986 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 4 Sep 2026 17:25:19 +0300 Subject: [PATCH 07/20] IGNITE-27088 BinaryWriter should use internal String#value --- .../internal/binary/BinaryWriterExImpl.java | 3 +- .../ignite/internal/binary/StringWriter.java | 2 +- .../ignite/IgniteCommonsSystemProperties.java | 5 +- .../direct/stream/DirectByteBufferStream.java | 109 +++++++++--------- 4 files changed, 62 insertions(+), 57 deletions(-) diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java index 12122b2acd01c..6c609b88e8435 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java @@ -42,6 +42,7 @@ import org.jetbrains.annotations.Nullable; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.ignite.IgniteCommonsSystemProperties.DFLT_ZERO_COPY; import static org.apache.ignite.IgniteCommonsSystemProperties.IGNITE_BINARY_STRING_ZERO_COPY; import static org.apache.ignite.internal.util.CommonUtils.MAX_ARRAY_SIZE; @@ -50,7 +51,7 @@ */ class BinaryWriterExImpl implements BinaryWriterEx { /** Zero-copy serialization enabled flag. */ - static final boolean ZERO_COPY = IgniteCommonsSystemProperties.getBoolean(IGNITE_BINARY_STRING_ZERO_COPY, true); + static final boolean ZERO_COPY = IgniteCommonsSystemProperties.getBoolean(IGNITE_BINARY_STRING_ZERO_COPY, DFLT_ZERO_COPY); /** Length: integer. */ private static final int LEN_INT = 4; diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java index 7615cf9af265e..e7a4b25ba4a17 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java @@ -302,7 +302,7 @@ public static byte[] latin1Value(String val) { * @param arr Array. * @return {@code True} if the array contains a byte with the sign bit set. */ - private static boolean hasNegatives(byte[] arr) { + public static boolean hasNegatives(byte[] arr) { if (HAS_NEGATIVES != null) { try { return (boolean)HAS_NEGATIVES.invokeExact(arr, 0, arr.length); diff --git a/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java b/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java index ab43cde019ec2..62e4b5f83187d 100644 --- a/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java +++ b/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java @@ -38,6 +38,9 @@ public class IgniteCommonsSystemProperties { /** Default value of {@link IgniteCommonsSystemProperties#IGNITE_USE_BINARY_ARRAYS}. */ public static final boolean DFLT_IGNITE_USE_BINARY_ARRAYS = false; + /** Default value of {@link IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY}. */ + public static final boolean DFLT_ZERO_COPY = true; + /** * Setting to {@code true} enables writing sensitive information in {@code toString()} output. */ @@ -132,7 +135,7 @@ public class IgniteCommonsSystemProperties { * Enables zero-copy UTF-8 serialization of {@link String} values. * Default value is {@code true}. */ - @SystemProperty(value = "Enables zero-copy UTF-8 serialization of String values", defaults = "false") + @SystemProperty(value = "Enables zero-copy UTF-8 serialization of String values", defaults = "" + DFLT_ZERO_COPY) public static final String IGNITE_BINARY_STRING_ZERO_COPY = "IGNITE_BINARY_STRING_ZERO_COPY"; /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java index 419e7835377fe..345f03953d049 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java @@ -17,57 +17,58 @@ package org.apache.ignite.internal.direct.stream; -import java.lang.reflect.Array; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.Collection; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.RandomAccess; -import java.util.UUID; -import java.util.function.BooleanSupplier; -import java.util.function.Supplier; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.IgniteException; -import org.apache.ignite.internal.binary.StringWriter; -import org.apache.ignite.internal.managers.communication.CompressedMessage; -import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; -import org.apache.ignite.internal.processors.cache.CacheObject; -import org.apache.ignite.internal.processors.cache.KeyCacheObject; -import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; -import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx; -import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; -import org.apache.ignite.internal.util.GridLongList; -import org.apache.ignite.internal.util.GridUnsafe; -import org.apache.ignite.internal.util.nio.MessageSerialization; -import org.apache.ignite.internal.util.tostring.GridToStringExclude; -import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.lang.IgniteProductVersion; -import org.apache.ignite.lang.IgniteUuid; -import org.apache.ignite.plugin.extensions.communication.Message; -import org.apache.ignite.plugin.extensions.communication.MessageArrayType; -import org.apache.ignite.plugin.extensions.communication.MessageCollectionType; -import org.apache.ignite.plugin.extensions.communication.MessageEnumType; -import org.apache.ignite.plugin.extensions.communication.MessageFactory; -import org.apache.ignite.plugin.extensions.communication.MessageMapType; -import org.apache.ignite.plugin.extensions.communication.MessageReader; -import org.apache.ignite.plugin.extensions.communication.MessageType; -import org.apache.ignite.plugin.extensions.communication.MessageWriter; -import org.jetbrains.annotations.Nullable; - -import static org.apache.ignite.internal.util.GridUnsafe.BIG_ENDIAN; -import static org.apache.ignite.internal.util.GridUnsafe.BYTE_ARR_OFF; -import static org.apache.ignite.internal.util.GridUnsafe.CHAR_ARR_OFF; -import static org.apache.ignite.internal.util.GridUnsafe.DOUBLE_ARR_OFF; -import static org.apache.ignite.internal.util.GridUnsafe.FLOAT_ARR_OFF; -import static org.apache.ignite.internal.util.GridUnsafe.INT_ARR_OFF; -import static org.apache.ignite.internal.util.GridUnsafe.LONG_ARR_OFF; -import static org.apache.ignite.internal.util.GridUnsafe.SHORT_ARR_OFF; + import java.lang.reflect.Array; + import java.nio.ByteBuffer; + import java.nio.charset.StandardCharsets; + import java.util.ArrayList; + import java.util.BitSet; + import java.util.Collection; + import java.util.EnumSet; + import java.util.HashSet; + import java.util.Iterator; + import java.util.List; + import java.util.Map; + import java.util.RandomAccess; + import java.util.UUID; + import java.util.function.BooleanSupplier; + import java.util.function.Supplier; + import org.apache.ignite.IgniteCheckedException; + import org.apache.ignite.IgniteException; + import org.apache.ignite.internal.binary.StringWriter; + import org.apache.ignite.internal.managers.communication.CompressedMessage; + import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; + import org.apache.ignite.internal.processors.cache.CacheObject; + import org.apache.ignite.internal.processors.cache.KeyCacheObject; + import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; + import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx; + import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; + import org.apache.ignite.internal.util.GridLongList; + import org.apache.ignite.internal.util.GridUnsafe; + import org.apache.ignite.internal.util.nio.MessageSerialization; + import org.apache.ignite.internal.util.tostring.GridToStringExclude; + import org.apache.ignite.internal.util.typedef.internal.S; + import org.apache.ignite.internal.util.typedef.internal.U; + import org.apache.ignite.lang.IgniteProductVersion; + import org.apache.ignite.lang.IgniteUuid; + import org.apache.ignite.plugin.extensions.communication.Message; + import org.apache.ignite.plugin.extensions.communication.MessageArrayType; + import org.apache.ignite.plugin.extensions.communication.MessageCollectionType; + import org.apache.ignite.plugin.extensions.communication.MessageEnumType; + import org.apache.ignite.plugin.extensions.communication.MessageFactory; + import org.apache.ignite.plugin.extensions.communication.MessageMapType; + import org.apache.ignite.plugin.extensions.communication.MessageReader; + import org.apache.ignite.plugin.extensions.communication.MessageType; + import org.apache.ignite.plugin.extensions.communication.MessageWriter; + import org.jetbrains.annotations.Nullable; + + import static org.apache.ignite.internal.util.GridUnsafe.BIG_ENDIAN; + import static org.apache.ignite.internal.util.GridUnsafe.BYTE_ARR_OFF; + import static org.apache.ignite.internal.util.GridUnsafe.CHAR_ARR_OFF; + import static org.apache.ignite.internal.util.GridUnsafe.DOUBLE_ARR_OFF; + import static org.apache.ignite.internal.util.GridUnsafe.FLOAT_ARR_OFF; + import static org.apache.ignite.internal.util.GridUnsafe.INT_ARR_OFF; + import static org.apache.ignite.internal.util.GridUnsafe.LONG_ARR_OFF; + import static org.apache.ignite.internal.util.GridUnsafe.SHORT_ARR_OFF; /** * Direct marshalling I/O stream. @@ -737,8 +738,8 @@ public void writeString(String val) { if (curStrBackingArr == null) { curStrBackingArr = StringWriter.latin1Value(val); - if (curStrBackingArr == null) - curStrBackingArr = val.getBytes(); + if (curStrBackingArr == null || StringWriter.hasNegatives(curStrBackingArr)) + curStrBackingArr = val.getBytes(StandardCharsets.UTF_8); } writeByteArray(curStrBackingArr); @@ -1367,7 +1368,7 @@ public boolean[] readBooleanArray() { public String readString() { byte[] arr = readByteArray(); - return arr != null ? new String(arr) : null; + return arr != null ? new String(arr, StandardCharsets.UTF_8) : null; } /** From 351fd2bed2eef674c7b9e0ccbd3b890e65280eed Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 4 Sep 2026 17:53:23 +0300 Subject: [PATCH 08/20] IGNITE-27088 BinaryWriter should use internal String#value --- .../direct/stream/DirectByteBufferStream.java | 104 +++++++++--------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java index 345f03953d049..d2176dfb6f617 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java @@ -17,58 +17,58 @@ package org.apache.ignite.internal.direct.stream; - import java.lang.reflect.Array; - import java.nio.ByteBuffer; - import java.nio.charset.StandardCharsets; - import java.util.ArrayList; - import java.util.BitSet; - import java.util.Collection; - import java.util.EnumSet; - import java.util.HashSet; - import java.util.Iterator; - import java.util.List; - import java.util.Map; - import java.util.RandomAccess; - import java.util.UUID; - import java.util.function.BooleanSupplier; - import java.util.function.Supplier; - import org.apache.ignite.IgniteCheckedException; - import org.apache.ignite.IgniteException; - import org.apache.ignite.internal.binary.StringWriter; - import org.apache.ignite.internal.managers.communication.CompressedMessage; - import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; - import org.apache.ignite.internal.processors.cache.CacheObject; - import org.apache.ignite.internal.processors.cache.KeyCacheObject; - import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; - import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx; - import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; - import org.apache.ignite.internal.util.GridLongList; - import org.apache.ignite.internal.util.GridUnsafe; - import org.apache.ignite.internal.util.nio.MessageSerialization; - import org.apache.ignite.internal.util.tostring.GridToStringExclude; - import org.apache.ignite.internal.util.typedef.internal.S; - import org.apache.ignite.internal.util.typedef.internal.U; - import org.apache.ignite.lang.IgniteProductVersion; - import org.apache.ignite.lang.IgniteUuid; - import org.apache.ignite.plugin.extensions.communication.Message; - import org.apache.ignite.plugin.extensions.communication.MessageArrayType; - import org.apache.ignite.plugin.extensions.communication.MessageCollectionType; - import org.apache.ignite.plugin.extensions.communication.MessageEnumType; - import org.apache.ignite.plugin.extensions.communication.MessageFactory; - import org.apache.ignite.plugin.extensions.communication.MessageMapType; - import org.apache.ignite.plugin.extensions.communication.MessageReader; - import org.apache.ignite.plugin.extensions.communication.MessageType; - import org.apache.ignite.plugin.extensions.communication.MessageWriter; - import org.jetbrains.annotations.Nullable; - - import static org.apache.ignite.internal.util.GridUnsafe.BIG_ENDIAN; - import static org.apache.ignite.internal.util.GridUnsafe.BYTE_ARR_OFF; - import static org.apache.ignite.internal.util.GridUnsafe.CHAR_ARR_OFF; - import static org.apache.ignite.internal.util.GridUnsafe.DOUBLE_ARR_OFF; - import static org.apache.ignite.internal.util.GridUnsafe.FLOAT_ARR_OFF; - import static org.apache.ignite.internal.util.GridUnsafe.INT_ARR_OFF; - import static org.apache.ignite.internal.util.GridUnsafe.LONG_ARR_OFF; - import static org.apache.ignite.internal.util.GridUnsafe.SHORT_ARR_OFF; +import java.lang.reflect.Array; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.RandomAccess; +import java.util.UUID; +import java.util.function.BooleanSupplier; +import java.util.function.Supplier; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.binary.StringWriter; +import org.apache.ignite.internal.managers.communication.CompressedMessage; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; +import org.apache.ignite.internal.processors.cache.CacheObject; +import org.apache.ignite.internal.processors.cache.KeyCacheObject; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx; +import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; +import org.apache.ignite.internal.util.GridLongList; +import org.apache.ignite.internal.util.GridUnsafe; +import org.apache.ignite.internal.util.nio.MessageSerialization; +import org.apache.ignite.internal.util.tostring.GridToStringExclude; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteProductVersion; +import org.apache.ignite.lang.IgniteUuid; +import org.apache.ignite.plugin.extensions.communication.Message; +import org.apache.ignite.plugin.extensions.communication.MessageArrayType; +import org.apache.ignite.plugin.extensions.communication.MessageCollectionType; +import org.apache.ignite.plugin.extensions.communication.MessageEnumType; +import org.apache.ignite.plugin.extensions.communication.MessageFactory; +import org.apache.ignite.plugin.extensions.communication.MessageMapType; +import org.apache.ignite.plugin.extensions.communication.MessageReader; +import org.apache.ignite.plugin.extensions.communication.MessageType; +import org.apache.ignite.plugin.extensions.communication.MessageWriter; +import org.jetbrains.annotations.Nullable; + +import static org.apache.ignite.internal.util.GridUnsafe.BIG_ENDIAN; +import static org.apache.ignite.internal.util.GridUnsafe.BYTE_ARR_OFF; +import static org.apache.ignite.internal.util.GridUnsafe.CHAR_ARR_OFF; +import static org.apache.ignite.internal.util.GridUnsafe.DOUBLE_ARR_OFF; +import static org.apache.ignite.internal.util.GridUnsafe.FLOAT_ARR_OFF; +import static org.apache.ignite.internal.util.GridUnsafe.INT_ARR_OFF; +import static org.apache.ignite.internal.util.GridUnsafe.LONG_ARR_OFF; +import static org.apache.ignite.internal.util.GridUnsafe.SHORT_ARR_OFF; /** * Direct marshalling I/O stream. From f7ce522c54014276654c4a60d78b3f67e521dcd9 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Fri, 4 Sep 2026 18:23:03 +0300 Subject: [PATCH 09/20] IGNITE-27088 BinaryWriter should use internal String#value --- .../test/java/org/apache/ignite/testsuites/ScriptTestSuite.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/calcite/src/test/java/org/apache/ignite/testsuites/ScriptTestSuite.java b/modules/calcite/src/test/java/org/apache/ignite/testsuites/ScriptTestSuite.java index 6c660b8097047..fbed0087a3aeb 100644 --- a/modules/calcite/src/test/java/org/apache/ignite/testsuites/ScriptTestSuite.java +++ b/modules/calcite/src/test/java/org/apache/ignite/testsuites/ScriptTestSuite.java @@ -72,6 +72,6 @@ * */ @RunWith(ScriptTestRunner.class) -@ScriptRunnerTestsEnvironment(scriptsRoot = "modules/calcite/src/test/sql", timeout = 180000, regex = "test_replace") +@ScriptRunnerTestsEnvironment(scriptsRoot = "modules/calcite/src/test/sql", timeout = 180000) public class ScriptTestSuite { } From fcdf587cca46004190579eb5724617e27038ae52 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Mon, 7 Sep 2026 15:06:23 +0300 Subject: [PATCH 10/20] IGNITE-27088 BinaryWriter should use internal String#value --- .../binary/JmhBinaryStringWriteBenchmark.java | 5 ++++- .../internal/binary/BinaryWriterExImpl.java | 17 ++--------------- .../ignite/internal/binary/StringWriter.java | 19 ++++++++++++++++++- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java index f029d50c8d76c..a2a3beb48c262 100644 --- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java @@ -156,7 +156,10 @@ public void tearDown() { public void writeString(Blackhole bh) { out.position(0); - StringWriter.write(str, out); + if (zeroCopy) + StringWriter.write(str, out); + else + StringWriter.writeStringLegacy(str, out); bh.consume(out.position()); } diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java index 6c609b88e8435..922c678fc4250 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java @@ -41,7 +41,6 @@ import org.apache.ignite.marshaller.Marshallers; import org.jetbrains.annotations.Nullable; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.ignite.IgniteCommonsSystemProperties.DFLT_ZERO_COPY; import static org.apache.ignite.IgniteCommonsSystemProperties.IGNITE_BINARY_STRING_ZERO_COPY; import static org.apache.ignite.internal.util.CommonUtils.MAX_ARRAY_SIZE; @@ -741,20 +740,8 @@ void writeBooleanField(@Nullable Boolean val) { out.writeByte(GridBinaryMarshaller.NULL); else if (ZERO_COPY) StringWriter.write(val, out); - else { - byte[] strArr; - - if (BinaryUtils.USE_STR_SERIALIZATION_VER_2) - strArr = BinaryUtils.strToUtf8Bytes(val); - else - strArr = val.getBytes(UTF_8); - - out.unsafeEnsure(1 + 4); - out.unsafeWriteByte(GridBinaryMarshaller.STRING); - out.unsafeWriteInt(strArr.length); - - out.writeByteArray(strArr); - } + else + StringWriter.writeStringLegacy(val, out); } /** {@inheritDoc} */ diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java index e7a4b25ba4a17..c65c2e3bea719 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java @@ -28,6 +28,7 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.ignite.internal.binary.BinaryWriterExImpl.ZERO_COPY; /** @@ -291,7 +292,7 @@ else if (!Character.isSurrogate(c)) { * @return Internal Latin-1 array of the string, * or {@code null} if the string is UTF-16 encoded or the internal layout of {@link String} is unknown. */ - public static byte[] latin1Value(String val) { + @Nullable public static byte[] latin1Value(String val) { if (STR_VALUE_OFF < 0 || GridUnsafe.getByteField(val, STR_CODER_OFF) != LATIN1) return null; @@ -371,4 +372,20 @@ private static IgniteBiTuple fieldsOffsets() { return null; } + + /** */ + public static void writeStringLegacy(@NotNull String val, BinaryOutputStream out) { + byte[] strArr; + + if (BinaryUtils.USE_STR_SERIALIZATION_VER_2) + strArr = BinaryUtils.strToUtf8Bytes(val); + else + strArr = val.getBytes(UTF_8); + + out.unsafeEnsure(1 + 4); + out.unsafeWriteByte(GridBinaryMarshaller.STRING); + out.unsafeWriteInt(strArr.length); + + out.writeByteArray(strArr); + } } From d765dedcbddf7c6cf0488f56033ff186e243f093 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Mon, 7 Sep 2026 17:33:43 +0300 Subject: [PATCH 11/20] IGNITE-27088 BinaryWriter should use internal String#value --- .../apache/ignite/internal/binary/StringWriter.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java index c65c2e3bea719..d62810a5105c9 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java @@ -339,13 +339,10 @@ private static IgniteBiTuple fieldsOffsets() { Field valField = String.class.getDeclaredField("value"); Field coderField = String.class.getDeclaredField("coder"); - // On JDK 8 the value field is a char[], only the generic encoder can be used. - if (valField.getType() == byte[].class && coderField.getType() == byte.class) { - return new IgniteBiTuple<>( - GridUnsafe.objectFieldOffset(valField), - GridUnsafe.objectFieldOffset(coderField) - ); - } + return new IgniteBiTuple<>( + GridUnsafe.objectFieldOffset(valField), + GridUnsafe.objectFieldOffset(coderField) + ); } catch (Throwable ignored) { // No-op. From b365c7fb7283360e4a51656b4a503e6d8de86797 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 9 Sep 2026 09:58:33 +0300 Subject: [PATCH 12/20] CheapString prototype --- .../ignite/internal/binary/BinaryUtils.java | 240 -------------- .../binary/BinaryWriterSchemaHolder.java | 2 +- .../internal/binary/BinaryImplUtils.java | 302 ++++++++++++++++++ .../internal/binary/BinaryObjectImpl.java | 22 +- .../binary/BinaryObjectOffheapImpl.java | 14 +- .../internal/binary/BinaryReaderExImpl.java | 28 +- .../binary/CrossObjectReferenceResolver.java | 22 +- .../internal/binary/ObjectDetachHelper.java | 4 +- .../builder/BinaryObjectBuilderImpl.java | 27 +- ...inarySerialiedFieldComparatorSelfTest.java | 2 +- 10 files changed, 363 insertions(+), 300 deletions(-) create mode 100644 modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryImplUtils.java diff --git a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index e10222a6593cd..4e0186f2d5f5d 100644 --- a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -86,7 +86,6 @@ import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.A; -import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.logger.NullLogger; import org.apache.ignite.marshaller.Marshallers; @@ -140,40 +139,6 @@ TreeSet.class, new BinaryTreeSetWriteReplacer() /** Class for SingletonList obtained at runtime. */ static final Class SINGLETON_LIST_CLS = Collections.singletonList(null).getClass(); - /** Flag: user type. */ - static final short FLAG_USR_TYP = 0x0001; - - /** Flag: only raw data exists. */ - static final short FLAG_HAS_SCHEMA = 0x0002; - - /** Flag indicating that object has raw data. */ - static final short FLAG_HAS_RAW = 0x0004; - - /** Flag: offsets take 1 byte. */ - static final short FLAG_OFFSET_ONE_BYTE = 0x0008; - - /** Flag: offsets take 2 bytes. */ - static final short FLAG_OFFSET_TWO_BYTES = 0x0010; - - /** Flag: compact footer, no field IDs. */ - public static final short FLAG_COMPACT_FOOTER = 0x0020; - - /** Flag: raw data contains .NET type information. Always 0 in Java. Keep it here for information only. */ - @SuppressWarnings("unused") - public static final short FLAG_CUSTOM_DOTNET_TYPE = 0x0040; - - /** Offset which fits into 1 byte. */ - static final int OFFSET_1 = 1; - - /** Offset which fits into 2 bytes. */ - static final int OFFSET_2 = 2; - - /** Offset which fits into 4 bytes. */ - static final int OFFSET_4 = 4; - - /** Field ID length. */ - static final int FIELD_ID_LEN = 4; - /** Whether to sort field in binary objects (doesn't affect Binarylizable). */ public static boolean FIELDS_SORTED_ORDER = IgniteCommonsSystemProperties.getBoolean(IgniteCommonsSystemProperties.IGNITE_BINARY_SORT_OBJECT_FIELDS); @@ -326,57 +291,6 @@ TreeSet.class, new BinaryTreeSetWriteReplacer() FIELD_TYPE_NAMES[GridBinaryMarshaller.BINARY_ENUM] = "Enum"; } - /** - * Check if user type flag is set. - * - * @param flags Flags. - * @return {@code True} if set. - */ - static boolean isUserType(short flags) { - return isFlagSet(flags, FLAG_USR_TYP); - } - - /** - * Check if raw-only flag is set. - * - * @param flags Flags. - * @return {@code True} if set. - */ - public static boolean hasSchema(short flags) { - return isFlagSet(flags, FLAG_HAS_SCHEMA); - } - - /** - * Check if raw-only flag is set. - * - * @param flags Flags. - * @return {@code True} if set. - */ - static boolean hasRaw(short flags) { - return isFlagSet(flags, FLAG_HAS_RAW); - } - - /** - * Check if "no-field-ids" flag is set. - * - * @param flags Flags. - * @return {@code True} if set. - */ - static boolean isCompactFooter(short flags) { - return isFlagSet(flags, FLAG_COMPACT_FOOTER); - } - - /** - * Check whether particular flag is set. - * - * @param flags Flags. - * @param flag Flag. - * @return {@code True} if flag is set in flags. - */ - static boolean isFlagSet(short flags, short flag) { - return (flags & flag) == flag; - } - /** * Schema initial ID. * @@ -784,160 +698,6 @@ public static int length(BinaryPositionReadable in, int start) { return in.readIntPositioned(start + GridBinaryMarshaller.TOTAL_LEN_POS); } - /** */ - static int dataStartRelative(BinaryPositionReadable in, int start) { - int typeId = in.readIntPositioned(start + GridBinaryMarshaller.TYPE_ID_POS); - - if (typeId == GridBinaryMarshaller.UNREGISTERED_TYPE_ID) { - // Gets the length of the type name which is stored as string. - int len = in.readIntPositioned(start + GridBinaryMarshaller.DFLT_HDR_LEN + /** object type */1); - - return GridBinaryMarshaller.DFLT_HDR_LEN + /** object type */1 + /** string length */ 4 + len; - } - else - return GridBinaryMarshaller.DFLT_HDR_LEN; - } - - /** - * Get footer start of the object. - * - * @param in Input stream. - * @param start Object start position inside the stream. - * @return Footer start. - */ - private static int footerStartRelative(BinaryPositionReadable in, int start) { - short flags = in.readShortPositioned(start + GridBinaryMarshaller.FLAGS_POS); - - if (hasSchema(flags)) - // Schema exists, use offset. - return in.readIntPositioned(start + GridBinaryMarshaller.SCHEMA_OR_RAW_OFF_POS); - else - // No schema, footer start equals to object end. - return length(in, start); - } - - /** - * Get object's footer. - * - * @param in Input stream. - * @param start Start position. - * @return Footer start. - */ - public static int footerStartAbsolute(BinaryPositionReadable in, int start) { - return footerStartRelative(in, start) + start; - } - - /** - * Get object's footer. - * - * @param in Input stream. - * @param start Start position. - * @return Footer. - */ - public static IgniteBiTuple footerAbsolute(BinaryPositionReadable in, int start) { - short flags = in.readShortPositioned(start + GridBinaryMarshaller.FLAGS_POS); - - int footerEnd = length(in, start); - - if (hasSchema(flags)) { - // Schema exists. - int footerStart = in.readIntPositioned(start + GridBinaryMarshaller.SCHEMA_OR_RAW_OFF_POS); - - if (hasRaw(flags)) - footerEnd -= 4; - - assert footerStart <= footerEnd; - - return F.t(start + footerStart, start + footerEnd); - } - else - // No schema. - return F.t(start + footerEnd, start + footerEnd); - } - - /** - * Get relative raw offset of the object. - * - * @param in Input stream. - * @param start Object start position inside the stream. - * @return Raw offset. - */ - private static int rawOffsetRelative(BinaryPositionReadable in, int start) { - short flags = in.readShortPositioned(start + GridBinaryMarshaller.FLAGS_POS); - - int len = length(in, start); - - if (hasSchema(flags)) { - // Schema exists. - if (hasRaw(flags)) - // Raw offset is set, it is at the very end of the object. - return in.readIntPositioned(start + len - 4); - else - // Raw offset is not set, so just return schema offset. - return in.readIntPositioned(start + GridBinaryMarshaller.SCHEMA_OR_RAW_OFF_POS); - } - else - // No schema, raw offset is located on schema offset position. - return in.readIntPositioned(start + GridBinaryMarshaller.SCHEMA_OR_RAW_OFF_POS); - } - - /** - * Get absolute raw offset of the object. - * - * @param in Input stream. - * @param start Object start position inside the stream. - * @return Raw offset. - */ - public static int rawOffsetAbsolute(BinaryPositionReadable in, int start) { - return start + rawOffsetRelative(in, start); - } - - /** - * Get offset length for the given flags. - * - * @param flags Flags. - * @return Offset size. - */ - public static int fieldOffsetLength(short flags) { - if ((flags & FLAG_OFFSET_ONE_BYTE) == FLAG_OFFSET_ONE_BYTE) - return OFFSET_1; - else if ((flags & FLAG_OFFSET_TWO_BYTES) == FLAG_OFFSET_TWO_BYTES) - return OFFSET_2; - else - return OFFSET_4; - } - - /** - * Get field ID length. - * - * @param flags Flags. - * @return Field ID length. - */ - public static int fieldIdLength(short flags) { - return isCompactFooter(flags) ? 0 : FIELD_ID_LEN; - } - - /** - * Get relative field offset. - * - * @param stream Stream. - * @param pos Position. - * @param fieldOffsetSize Field offset size. - * @return Relative field offset. - */ - public static int fieldOffsetRelative(BinaryPositionReadable stream, int pos, int fieldOffsetSize) { - int res; - - if (fieldOffsetSize == OFFSET_1) - res = (int)stream.readBytePositioned(pos) & 0xFF; - else if (fieldOffsetSize == OFFSET_2) - res = (int)stream.readShortPositioned(pos) & 0xFFFF; - else - res = stream.readIntPositioned(pos); - - return res; - } - /** * Merge old and new metas. * diff --git a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryWriterSchemaHolder.java b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryWriterSchemaHolder.java index cd2d8c5d76124..8e5eab4f9f6ee 100644 --- a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryWriterSchemaHolder.java +++ b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryWriterSchemaHolder.java @@ -134,7 +134,7 @@ else if (lastOffset < MAX_OFFSET_2) { out.unsafeWriteInt(data[curIdx++]); } - res = BinaryUtils.OFFSET_4; + res = BinaryImplUtils.OFFSET_4; } } diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryImplUtils.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryImplUtils.java new file mode 100644 index 0000000000000..2e23d5ca265dc --- /dev/null +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryImplUtils.java @@ -0,0 +1,302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.internal.binary; + +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.lang.IgniteBiTuple; + +import static org.apache.ignite.internal.binary.BinaryUtils.length; + +/** + * Binary utils used only in implementation. + */ +public class BinaryImplUtils { + /** Flag: user type. */ + static final short FLAG_USR_TYP = 0x0001; + + /** Flag: only raw data exists. */ + static final short FLAG_HAS_SCHEMA = 0x0002; + + /** Flag indicating that object has raw data. */ + static final short FLAG_HAS_RAW = 0x0004; + + /** Flag: offsets take 1 byte. */ + static final short FLAG_OFFSET_ONE_BYTE = 0x0008; + + /** Flag: offsets take 2 bytes. */ + static final short FLAG_OFFSET_TWO_BYTES = 0x0010; + + /** Flag: compact footer, no field IDs. */ + public static final short FLAG_COMPACT_FOOTER = 0x0020; + + /** Flag: raw data contains .NET type information. Always 0 in Java. Keep it here for information only. */ + @SuppressWarnings("unused") + public static final short FLAG_CUSTOM_DOTNET_TYPE = 0x0040; + + /** Offset which fits into 1 byte. */ + static final int OFFSET_1 = 1; + + /** Offset which fits into 2 bytes. */ + static final int OFFSET_2 = 2; + + /** Offset which fits into 4 bytes. */ + static final int OFFSET_4 = 4; + + /** Field ID length. */ + static final int FIELD_ID_LEN = 4; + + /** {@code true} if serialized value of this type cannot contain references to objects. */ + private static final boolean[] PLAIN_TYPE_FLAG = new boolean[102]; + + static { + for (byte b : new byte[] { + GridBinaryMarshaller.BYTE, GridBinaryMarshaller.SHORT, GridBinaryMarshaller.INT, GridBinaryMarshaller.LONG, + GridBinaryMarshaller.FLOAT, GridBinaryMarshaller.DOUBLE, GridBinaryMarshaller.CHAR, GridBinaryMarshaller.BOOLEAN, + GridBinaryMarshaller.DECIMAL, GridBinaryMarshaller.STRING, GridBinaryMarshaller.UUID, GridBinaryMarshaller.DATE, + GridBinaryMarshaller.TIMESTAMP, GridBinaryMarshaller.TIME, GridBinaryMarshaller.BYTE_ARR, GridBinaryMarshaller.SHORT_ARR, + GridBinaryMarshaller.INT_ARR, GridBinaryMarshaller.LONG_ARR, GridBinaryMarshaller.FLOAT_ARR, GridBinaryMarshaller.DOUBLE_ARR, + GridBinaryMarshaller.TIME_ARR, GridBinaryMarshaller.CHAR_ARR, GridBinaryMarshaller.BOOLEAN_ARR, + GridBinaryMarshaller.DECIMAL_ARR, GridBinaryMarshaller.STRING_ARR, GridBinaryMarshaller.UUID_ARR, GridBinaryMarshaller.DATE_ARR, + GridBinaryMarshaller.TIMESTAMP_ARR, GridBinaryMarshaller.ENUM, GridBinaryMarshaller.ENUM_ARR, GridBinaryMarshaller.NULL}) { + + PLAIN_TYPE_FLAG[b] = true; + } + } + + /** + * Check if user type flag is set. + * + * @param flags Flags. + * @return {@code True} if set. + */ + static boolean isUserType(short flags) { + return isFlagSet(flags, FLAG_USR_TYP); + } + + /** + * Check if raw-only flag is set. + * + * @param flags Flags. + * @return {@code True} if set. + */ + public static boolean hasSchema(short flags) { + return isFlagSet(flags, FLAG_HAS_SCHEMA); + } + + /** + * Check if raw-only flag is set. + * + * @param flags Flags. + * @return {@code True} if set. + */ + static boolean hasRaw(short flags) { + return isFlagSet(flags, FLAG_HAS_RAW); + } + + /** + * Check if "no-field-ids" flag is set. + * + * @param flags Flags. + * @return {@code True} if set. + */ + static boolean isCompactFooter(short flags) { + return isFlagSet(flags, FLAG_COMPACT_FOOTER); + } + + /** + * Check whether particular flag is set. + * + * @param flags Flags. + * @param flag Flag. + * @return {@code True} if flag is set in flags. + */ + static boolean isFlagSet(short flags, short flag) { + return (flags & flag) == flag; + } + + /** */ + static int dataStartRelative(BinaryPositionReadable in, int start) { + int typeId = in.readIntPositioned(start + GridBinaryMarshaller.TYPE_ID_POS); + + if (typeId == GridBinaryMarshaller.UNREGISTERED_TYPE_ID) { + // Gets the length of the type name which is stored as string. + int len = in.readIntPositioned(start + GridBinaryMarshaller.DFLT_HDR_LEN + /** object type */1); + + return GridBinaryMarshaller.DFLT_HDR_LEN + /** object type */1 + /** string length */ 4 + len; + } + else + return GridBinaryMarshaller.DFLT_HDR_LEN; + } + + /** + * Get footer start of the object. + * + * @param in Input stream. + * @param start Object start position inside the stream. + * @return Footer start. + */ + private static int footerStartRelative(BinaryPositionReadable in, int start) { + short flags = in.readShortPositioned(start + GridBinaryMarshaller.FLAGS_POS); + + if (hasSchema(flags)) + // Schema exists, use offset. + return in.readIntPositioned(start + GridBinaryMarshaller.SCHEMA_OR_RAW_OFF_POS); + else + // No schema, footer start equals to object end. + return length(in, start); + } + + /** + * Get object's footer. + * + * @param in Input stream. + * @param start Start position. + * @return Footer start. + */ + public static int footerStartAbsolute(BinaryPositionReadable in, int start) { + return footerStartRelative(in, start) + start; + } + + /** + * Get object's footer. + * + * @param in Input stream. + * @param start Start position. + * @return Footer. + */ + public static IgniteBiTuple footerAbsolute(BinaryPositionReadable in, int start) { + short flags = in.readShortPositioned(start + GridBinaryMarshaller.FLAGS_POS); + + int footerEnd = length(in, start); + + if (hasSchema(flags)) { + // Schema exists. + int footerStart = in.readIntPositioned(start + GridBinaryMarshaller.SCHEMA_OR_RAW_OFF_POS); + + if (hasRaw(flags)) + footerEnd -= 4; + + assert footerStart <= footerEnd; + + return F.t(start + footerStart, start + footerEnd); + } + else + // No schema. + return F.t(start + footerEnd, start + footerEnd); + } + + /** + * Get relative raw offset of the object. + * + * @param in Input stream. + * @param start Object start position inside the stream. + * @return Raw offset. + */ + private static int rawOffsetRelative(BinaryPositionReadable in, int start) { + short flags = in.readShortPositioned(start + GridBinaryMarshaller.FLAGS_POS); + + int len = length(in, start); + + if (hasSchema(flags)) { + // Schema exists. + if (hasRaw(flags)) + // Raw offset is set, it is at the very end of the object. + return in.readIntPositioned(start + len - 4); + else + // Raw offset is not set, so just return schema offset. + return in.readIntPositioned(start + GridBinaryMarshaller.SCHEMA_OR_RAW_OFF_POS); + } + else + // No schema, raw offset is located on schema offset position. + return in.readIntPositioned(start + GridBinaryMarshaller.SCHEMA_OR_RAW_OFF_POS); + } + + /** + * Get absolute raw offset of the object. + * + * @param in Input stream. + * @param start Object start position inside the stream. + * @return Raw offset. + */ + public static int rawOffsetAbsolute(BinaryPositionReadable in, int start) { + return start + rawOffsetRelative(in, start); + } + + /** + * Get offset length for the given flags. + * + * @param flags Flags. + * @return Offset size. + */ + public static int fieldOffsetLength(short flags) { + if ((flags & FLAG_OFFSET_ONE_BYTE) == FLAG_OFFSET_ONE_BYTE) + return OFFSET_1; + else if ((flags & FLAG_OFFSET_TWO_BYTES) == FLAG_OFFSET_TWO_BYTES) + return OFFSET_2; + else + return OFFSET_4; + } + + /** + * Get field ID length. + * + * @param flags Flags. + * @return Field ID length. + */ + public static int fieldIdLength(short flags) { + return isCompactFooter(flags) ? 0 : FIELD_ID_LEN; + } + + /** + * Get relative field offset. + * + * @param stream Stream. + * @param pos Position. + * @param fieldOffsetSize Field offset size. + * @return Relative field offset. + */ + public static int fieldOffsetRelative(BinaryPositionReadable stream, int pos, int fieldOffsetSize) { + int res; + + if (fieldOffsetSize == OFFSET_1) + res = (int)stream.readBytePositioned(pos) & 0xFF; + else if (fieldOffsetSize == OFFSET_2) + res = (int)stream.readShortPositioned(pos) & 0xFFFF; + else + res = stream.readIntPositioned(pos); + + return res; + } + + /** + * @return {@code true} if content of serialized value cannot contain references to other object. + */ + public static boolean isPlainType(int type) { + return type > 0 && type < PLAIN_TYPE_FLAG.length && PLAIN_TYPE_FLAG[type]; + } + + /** + * Checks whether an array type values can or can not contain references to other object. + * + * @param type Array type. + * @return {@code true} if content of serialized array value cannot contain references to other object. + */ + public static boolean isPlainArrayType(int type) { + return (type >= GridBinaryMarshaller.BYTE_ARR && type <= GridBinaryMarshaller.DATE_ARR) + || type == GridBinaryMarshaller.TIMESTAMP_ARR || type == GridBinaryMarshaller.TIME_ARR; + } +} diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java index 7dd2bebdf86af..a0a9c9c9c90af 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java @@ -345,7 +345,7 @@ public void context(BinaryContext ctx) { @Override public boolean isFlagSet(short flag) { short flags = BinaryPrimitives.readShort(arr, start + GridBinaryMarshaller.FLAGS_POS); - return BinaryUtils.isFlagSet(flags, flag); + return BinaryImplUtils.isFlagSet(flags, flag); } /** {@inheritDoc} */ @@ -406,7 +406,7 @@ public void context(BinaryContext ctx) { @Override public int footerStartOffset() { short flags = BinaryPrimitives.readShort(arr, start + GridBinaryMarshaller.FLAGS_POS); - if (!BinaryUtils.hasSchema(flags)) + if (!BinaryImplUtils.hasSchema(flags)) return start + length(); return start + BinaryPrimitives.readInt(arr, start + GridBinaryMarshaller.SCHEMA_OR_RAW_OFF_POS); @@ -424,16 +424,16 @@ public void context(BinaryContext ctx) { short flags = BinaryPrimitives.readShort(arr, start + GridBinaryMarshaller.FLAGS_POS); - int fieldIdLen = BinaryUtils.isCompactFooter(flags) ? 0 : BinaryUtils.FIELD_ID_LEN; - int fieldOffLen = BinaryUtils.fieldOffsetLength(flags); + int fieldIdLen = BinaryImplUtils.isCompactFooter(flags) ? 0 : BinaryImplUtils.FIELD_ID_LEN; + int fieldOffLen = BinaryImplUtils.fieldOffsetLength(flags); int fieldOffsetPos = start + schemaOff + order * (fieldIdLen + fieldOffLen) + fieldIdLen; int fieldPos; - if (fieldOffLen == BinaryUtils.OFFSET_1) + if (fieldOffLen == BinaryImplUtils.OFFSET_1) fieldPos = start + ((int)BinaryPrimitives.readByte(arr, fieldOffsetPos) & 0xFF); - else if (fieldOffLen == BinaryUtils.OFFSET_2) + else if (fieldOffLen == BinaryImplUtils.OFFSET_2) fieldPos = start + ((int)BinaryPrimitives.readShort(arr, fieldOffsetPos) & 0xFFFF); else fieldPos = start + BinaryPrimitives.readInt(arr, fieldOffsetPos); @@ -570,16 +570,16 @@ else if (fieldOffLen == BinaryUtils.OFFSET_2) short flags = BinaryPrimitives.readShort(arr, start + GridBinaryMarshaller.FLAGS_POS); - int fieldIdLen = BinaryUtils.isCompactFooter(flags) ? 0 : BinaryUtils.FIELD_ID_LEN; - int fieldOffsetLen = BinaryUtils.fieldOffsetLength(flags); + int fieldIdLen = BinaryImplUtils.isCompactFooter(flags) ? 0 : BinaryImplUtils.FIELD_ID_LEN; + int fieldOffsetLen = BinaryImplUtils.fieldOffsetLength(flags); int fieldOffsetPos = start + schemaOffset + order * (fieldIdLen + fieldOffsetLen) + fieldIdLen; int fieldPos; - if (fieldOffsetLen == BinaryUtils.OFFSET_1) + if (fieldOffsetLen == BinaryImplUtils.OFFSET_1) fieldPos = start + ((int)BinaryPrimitives.readByte(arr, fieldOffsetPos) & 0xFF); - else if (fieldOffsetLen == BinaryUtils.OFFSET_2) + else if (fieldOffsetLen == BinaryImplUtils.OFFSET_2) fieldPos = start + ((int)BinaryPrimitives.readShort(arr, fieldOffsetPos) & 0xFFFF); else fieldPos = start + BinaryPrimitives.readInt(arr, fieldOffsetPos); @@ -728,7 +728,7 @@ else if (fieldOffsetLen == BinaryUtils.OFFSET_2) @Override public boolean hasSchema() { short flags = BinaryPrimitives.readShort(arr, start + GridBinaryMarshaller.FLAGS_POS); - return BinaryUtils.hasSchema(flags); + return BinaryImplUtils.hasSchema(flags); } /** {@inheritDoc} */ diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java index e2ac1b89b4e42..665607bc109a0 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java @@ -117,7 +117,7 @@ public BinaryObjectOffheapImpl(BinaryContext ctx, long ptr, int start, int size) @Override public boolean hasSchema() { short flags = BinaryPrimitives.readShort(ptr, start + GridBinaryMarshaller.FLAGS_POS); - return BinaryUtils.hasSchema(flags); + return BinaryImplUtils.hasSchema(flags); } /** {@inheritDoc} */ @@ -175,7 +175,7 @@ public BinaryObjectOffheapImpl(BinaryContext ctx, long ptr, int start, int size) @Override public boolean isFlagSet(short flag) { short flags = BinaryPrimitives.readShort(ptr, start + GridBinaryMarshaller.FLAGS_POS); - return BinaryUtils.isFlagSet(flags, flag); + return BinaryImplUtils.isFlagSet(flags, flag); } /** {@inheritDoc} */ @@ -215,7 +215,7 @@ public BinaryObjectOffheapImpl(BinaryContext ctx, long ptr, int start, int size) @Override public int footerStartOffset() { short flags = BinaryPrimitives.readShort(ptr, start + GridBinaryMarshaller.FLAGS_POS); - if (!BinaryUtils.hasSchema(flags)) + if (!BinaryImplUtils.hasSchema(flags)) return start + length(); return start + BinaryPrimitives.readInt(ptr, start + GridBinaryMarshaller.SCHEMA_OR_RAW_OFF_POS); @@ -233,16 +233,16 @@ public BinaryObjectOffheapImpl(BinaryContext ctx, long ptr, int start, int size) short flags = BinaryPrimitives.readShort(ptr, start + GridBinaryMarshaller.FLAGS_POS); - int fieldIdLen = BinaryUtils.isCompactFooter(flags) ? 0 : BinaryUtils.FIELD_ID_LEN; - int fieldOffLen = BinaryUtils.fieldOffsetLength(flags); + int fieldIdLen = BinaryImplUtils.isCompactFooter(flags) ? 0 : BinaryImplUtils.FIELD_ID_LEN; + int fieldOffLen = BinaryImplUtils.fieldOffsetLength(flags); int fieldOffsetPos = start + schemaOff + order * (fieldIdLen + fieldOffLen) + fieldIdLen; int fieldPos; - if (fieldOffLen == BinaryUtils.OFFSET_1) + if (fieldOffLen == BinaryImplUtils.OFFSET_1) fieldPos = start + ((int)BinaryPrimitives.readByte(ptr, fieldOffsetPos) & 0xFF); - else if (fieldOffLen == BinaryUtils.OFFSET_2) + else if (fieldOffLen == BinaryImplUtils.OFFSET_2) fieldPos = start + ((int)BinaryPrimitives.readShort(ptr, fieldOffsetPos) & 0xFFFF); else fieldPos = start + BinaryPrimitives.readInt(ptr, fieldOffsetPos); diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java index 888494a185c6e..1fe574d811e48 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java @@ -233,16 +233,16 @@ public BinaryReaderExImpl(BinaryContext ctx, int offset = in.readInt(); // Get trivial flag values. - userType = BinaryUtils.isUserType(flags); - fieldIdLen = BinaryUtils.fieldIdLength(flags); - fieldOffLen = BinaryUtils.fieldOffsetLength(flags); + userType = BinaryImplUtils.isUserType(flags); + fieldIdLen = BinaryImplUtils.fieldIdLength(flags); + fieldOffLen = BinaryImplUtils.fieldOffsetLength(flags); // Calculate footer borders and raw offset. - if (BinaryUtils.hasSchema(flags)) { + if (BinaryImplUtils.hasSchema(flags)) { // Schema exists. footerStart = start + offset; - if (BinaryUtils.hasRaw(flags)) { + if (BinaryImplUtils.hasRaw(flags)) { footerLen = len - offset; rawOff = start + in.readIntPositioned(start + len - 4); } @@ -256,7 +256,7 @@ public BinaryReaderExImpl(BinaryContext ctx, footerStart = start + len; footerLen = 0; - if (BinaryUtils.hasRaw(flags)) + if (BinaryImplUtils.hasRaw(flags)) rawOff = start + offset; else rawOff = start + len; @@ -286,7 +286,7 @@ public BinaryReaderExImpl(BinaryContext ctx, } mapper = userType ? ctx.userTypeMapper(typeId) : BinaryContext.defaultMapper(); - schema = BinaryUtils.hasSchema(flags) ? getOrCreateSchema() : null; + schema = BinaryImplUtils.hasSchema(flags) ? getOrCreateSchema() : null; } else { dataStart = 0; @@ -1989,7 +1989,7 @@ private int fieldId(String name) { BinarySchema schema = ctx.schemaRegistry(typeId).schema(schemaId); if (schema == null) { - if (fieldIdLen != BinaryUtils.FIELD_ID_LEN) { + if (fieldIdLen != BinaryImplUtils.FIELD_ID_LEN) { BinaryTypeImpl type = (BinaryTypeImpl)ctx.metadata(typeId, schemaId); BinaryMetadata meta = type != null ? type.metadata() : null; @@ -2041,7 +2041,7 @@ private int fieldId(String name) { * @return Schema. */ private BinarySchema createSchema() { - assert fieldIdLen == BinaryUtils.FIELD_ID_LEN; + assert fieldIdLen == BinaryImplUtils.FIELD_ID_LEN; BinarySchema.Builder builder = BinarySchema.Builder.newBuilder(); @@ -2053,7 +2053,7 @@ private BinarySchema createSchema() { builder.addField(fieldId); - searchPos += BinaryUtils.FIELD_ID_LEN + fieldOffLen; + searchPos += BinaryImplUtils.FIELD_ID_LEN + fieldOffLen; } return builder.build(); @@ -2184,7 +2184,7 @@ private boolean trySetUserFieldPosition(int order) { if (order != BinarySchema.ORDER_NOT_FOUND) { int offsetPos = footerStart + order * (fieldIdLen + fieldOffLen) + fieldIdLen; - int pos = start + BinaryUtils.fieldOffsetRelative(in, offsetPos, fieldOffLen); + int pos = start + BinaryImplUtils.fieldOffsetRelative(in, offsetPos, fieldOffLen); streamPosition(pos); @@ -2202,7 +2202,7 @@ private boolean trySetUserFieldPosition(int order) { */ private boolean trySetSystemFieldPosition(int id) { // System types are never written with compact footers because they do not have metadata. - assert fieldIdLen == BinaryUtils.FIELD_ID_LEN; + assert fieldIdLen == BinaryImplUtils.FIELD_ID_LEN; int searchPos = footerStart; int searchTail = searchPos + footerLen; @@ -2214,7 +2214,7 @@ private boolean trySetSystemFieldPosition(int id) { int id0 = in.readIntPositioned(searchPos); if (id0 == id) { - int pos = start + BinaryUtils.fieldOffsetRelative(in, searchPos + BinaryUtils.FIELD_ID_LEN, + int pos = start + BinaryImplUtils.fieldOffsetRelative(in, searchPos + BinaryImplUtils.FIELD_ID_LEN, fieldOffLen); streamPosition(pos); @@ -2222,7 +2222,7 @@ private boolean trySetSystemFieldPosition(int id) { return true; } - searchPos += BinaryUtils.FIELD_ID_LEN + fieldOffLen; + searchPos += BinaryImplUtils.FIELD_ID_LEN + fieldOffLen; } } diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/CrossObjectReferenceResolver.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/CrossObjectReferenceResolver.java index 845f057fe8f3d..42815f6b8f3b0 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/CrossObjectReferenceResolver.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/CrossObjectReferenceResolver.java @@ -21,18 +21,18 @@ import java.util.Map; import org.apache.ignite.internal.binary.streams.BinaryOutputStream; -import static org.apache.ignite.internal.binary.BinaryUtils.FLAG_OFFSET_ONE_BYTE; -import static org.apache.ignite.internal.binary.BinaryUtils.FLAG_OFFSET_TWO_BYTES; -import static org.apache.ignite.internal.binary.BinaryUtils.OFFSET_1; -import static org.apache.ignite.internal.binary.BinaryUtils.OFFSET_2; -import static org.apache.ignite.internal.binary.BinaryUtils.dataStartRelative; -import static org.apache.ignite.internal.binary.BinaryUtils.fieldOffsetLength; -import static org.apache.ignite.internal.binary.BinaryUtils.footerStartAbsolute; -import static org.apache.ignite.internal.binary.BinaryUtils.hasRaw; -import static org.apache.ignite.internal.binary.BinaryUtils.hasSchema; -import static org.apache.ignite.internal.binary.BinaryUtils.isCompactFooter; +import static org.apache.ignite.internal.binary.BinaryImplUtils.FLAG_OFFSET_ONE_BYTE; +import static org.apache.ignite.internal.binary.BinaryImplUtils.FLAG_OFFSET_TWO_BYTES; +import static org.apache.ignite.internal.binary.BinaryImplUtils.OFFSET_1; +import static org.apache.ignite.internal.binary.BinaryImplUtils.OFFSET_2; +import static org.apache.ignite.internal.binary.BinaryImplUtils.dataStartRelative; +import static org.apache.ignite.internal.binary.BinaryImplUtils.fieldOffsetLength; +import static org.apache.ignite.internal.binary.BinaryImplUtils.footerStartAbsolute; +import static org.apache.ignite.internal.binary.BinaryImplUtils.hasRaw; +import static org.apache.ignite.internal.binary.BinaryImplUtils.hasSchema; +import static org.apache.ignite.internal.binary.BinaryImplUtils.isCompactFooter; +import static org.apache.ignite.internal.binary.BinaryImplUtils.rawOffsetAbsolute; import static org.apache.ignite.internal.binary.BinaryUtils.length; -import static org.apache.ignite.internal.binary.BinaryUtils.rawOffsetAbsolute; import static org.apache.ignite.internal.binary.GridBinaryMarshaller.DFLT_HDR_LEN; import static org.apache.ignite.internal.binary.GridBinaryMarshaller.FLAGS_POS; import static org.apache.ignite.internal.binary.GridBinaryMarshaller.HASH_CODE_POS; diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java index 1c9edee42b18e..d5e40860186e0 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/ObjectDetachHelper.java @@ -21,9 +21,9 @@ import org.apache.ignite.internal.binary.streams.BinaryOutputStream; import org.apache.ignite.internal.binary.streams.BinaryStreams; -import static org.apache.ignite.internal.binary.BinaryUtils.dataStartRelative; +import static org.apache.ignite.internal.binary.BinaryImplUtils.dataStartRelative; +import static org.apache.ignite.internal.binary.BinaryImplUtils.rawOffsetAbsolute; import static org.apache.ignite.internal.binary.BinaryUtils.length; -import static org.apache.ignite.internal.binary.BinaryUtils.rawOffsetAbsolute; /** */ class ObjectDetachHelper { diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java index 65c312283032b..9a19d3dcb994c 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java @@ -33,6 +33,7 @@ import org.apache.ignite.internal.binary.BinariesFactoryImpl; import org.apache.ignite.internal.binary.BinaryContext; import org.apache.ignite.internal.binary.BinaryFieldMetadata; +import org.apache.ignite.internal.binary.BinaryImplUtils; import org.apache.ignite.internal.binary.BinaryObjectEx; import org.apache.ignite.internal.binary.BinaryObjectImpl; import org.apache.ignite.internal.binary.BinaryUtils; @@ -231,16 +232,16 @@ void serializeTo(BinaryWriterEx writer, BinaryBuilderSerializer serializer) { assignedFldsById = Collections.emptyMap(); // Get footer details. - int fieldIdLen = BinaryUtils.fieldIdLength(flags); - int fieldOffsetLen = BinaryUtils.fieldOffsetLength(flags); + int fieldIdLen = BinaryImplUtils.fieldIdLength(flags); + int fieldOffsetLen = BinaryImplUtils.fieldOffsetLength(flags); - IgniteBiTuple footer = BinaryUtils.footerAbsolute(reader, start); + IgniteBiTuple footer = BinaryImplUtils.footerAbsolute(reader, start); int footerPos = footer.get1(); int footerEnd = footer.get2(); // Get raw position. - int rawPos = BinaryUtils.rawOffsetAbsolute(reader, start); + int rawPos = BinaryImplUtils.rawOffsetAbsolute(reader, start); // Position reader on data. reader.position(start + hdrLen); @@ -268,7 +269,7 @@ void serializeTo(BinaryWriterEx writer, BinaryBuilderSerializer serializer) { else { int type = fieldLen != 0 ? reader.readByte(0) : 0; - if (fieldLen != 0 && !BinaryUtils.isPlainArrayType(type) && BinaryUtils.isPlainType(type)) { + if (fieldLen != 0 && !BinaryImplUtils.isPlainArrayType(type) && BinaryImplUtils.isPlainType(type)) { writer.writeFieldId(fieldId); writer.write(reader.array(), reader.position(), fieldLen); @@ -322,8 +323,8 @@ else if (readCache == null) { if (reader != null) { // Write raw data if any. - int rawOff = BinaryUtils.rawOffsetAbsolute(reader, start); - int footerStart = BinaryUtils.footerStartAbsolute(reader, start); + int rawOff = BinaryImplUtils.rawOffsetAbsolute(reader, start); + int footerStart = BinaryImplUtils.footerStartAbsolute(reader, start); if (rawOff < footerStart) { writer.rawWriter(); @@ -432,7 +433,7 @@ else if (!nullFieldVal) { private IgniteBiTuple fieldPositionAndLength(int footerPos, int footerEnd, int rawPos, int fieldIdLen, int fieldOffsetLen) { // Get field offset first. - int fieldOffset = BinaryUtils.fieldOffsetRelative(reader, footerPos + fieldIdLen, fieldOffsetLen); + int fieldOffset = BinaryImplUtils.fieldOffsetRelative(reader, footerPos + fieldIdLen, fieldOffsetLen); int fieldPos = start + fieldOffset; // Get field length. @@ -443,7 +444,7 @@ private IgniteBiTuple fieldPositionAndLength(int footerPos, in fieldLen = rawPos - fieldPos; else { // Field is somewhere in the middle, get difference with the next offset. - int nextFieldOffset = BinaryUtils.fieldOffsetRelative(reader, + int nextFieldOffset = BinaryImplUtils.fieldOffsetRelative(reader, footerPos + fieldIdLen + fieldOffsetLen + fieldIdLen, fieldOffsetLen); fieldLen = nextFieldOffset - fieldOffset; @@ -459,17 +460,17 @@ private void ensureReadCacheInit() { assert reader != null; if (readCache == null) { - int fieldIdLen = BinaryUtils.fieldIdLength(flags); - int fieldOffsetLen = BinaryUtils.fieldOffsetLength(flags); + int fieldIdLen = BinaryImplUtils.fieldIdLength(flags); + int fieldOffsetLen = BinaryImplUtils.fieldOffsetLength(flags); Map readCache = new HashMap<>(); - IgniteBiTuple footer = BinaryUtils.footerAbsolute(reader, start); + IgniteBiTuple footer = BinaryImplUtils.footerAbsolute(reader, start); int footerPos = footer.get1(); int footerEnd = footer.get2(); - int rawPos = BinaryUtils.rawOffsetAbsolute(reader, start); + int rawPos = BinaryImplUtils.rawOffsetAbsolute(reader, start); int idx = 0; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerialiedFieldComparatorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerialiedFieldComparatorSelfTest.java index bacc6fb1e3490..868e35223e4dd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerialiedFieldComparatorSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerialiedFieldComparatorSelfTest.java @@ -504,7 +504,7 @@ private BinarySerializedFieldComparator comparator(BinaryObjectExImpl obj) { : BinaryPrimitives.readShort(arr, start + GridBinaryMarshaller.FLAGS_POS); int fieldIdLen = BinaryUtils.isCompactFooter(flags) ? 0 : BinaryUtils.FIELD_ID_LEN; - int fieldOffLen = BinaryUtils.fieldOffsetLength(flags); + int fieldOffLen = BinaryImplUtils.fieldOffsetLength(flags); int orderBase = start + schemaOff + fieldIdLen; int orderMultiplier = fieldIdLen + fieldOffLen; From 810cf79480317f35bd68cfa312060c37fdc0d525 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 9 Sep 2026 10:23:59 +0300 Subject: [PATCH 13/20] IGNITE-29044 Move BinaryWriterSchemaHolder to binary-impl --- .../org/apache/ignite/internal/binary/BinariesFactory.java | 3 +-- .../java/org/apache/ignite/internal/binary/BinaryUtils.java | 4 ++-- .../apache/ignite/internal/binary/BinariesFactoryImpl.java | 4 ++-- .../apache/ignite/internal/binary/BinaryWriterExImpl.java | 4 ++-- .../ignite/internal/binary/BinaryWriterSchemaHolder.java | 0 .../org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java | 5 ++--- .../internal/processors/odbc/ClientListenerNioListener.java | 2 +- .../ignite/internal/client/thin/events/FakeIgniteServer.java | 2 +- .../ignite/platform/PlatformGetInternalCachesTask.java | 2 +- .../ignite/internal/client/thin/ClientAtomicLongImpl.java | 2 +- .../internal/client/thin/ClientCacheAffinityContext.java | 2 +- .../internal/client/thin/ClientCacheAffinityMapping.java | 2 +- .../org/apache/ignite/internal/client/thin/ClientUtils.java | 4 ++-- .../apache/ignite/internal/client/thin/TcpClientCache.java | 2 +- .../apache/ignite/internal/client/thin/TcpClientChannel.java | 2 +- .../ignite/internal/client/thin/TcpClientTransactions.java | 2 +- .../apache/ignite/internal/client/thin/TcpIgniteClient.java | 2 +- 17 files changed, 21 insertions(+), 23 deletions(-) rename modules/binary/{api => impl}/src/main/java/org/apache/ignite/internal/binary/BinaryWriterSchemaHolder.java (100%) diff --git a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinariesFactory.java b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinariesFactory.java index 3538c2b452f1d..c06e669ee2887 100644 --- a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinariesFactory.java +++ b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinariesFactory.java @@ -90,10 +90,9 @@ public BinaryReaderEx reader(BinaryContext ctx, /** * @param ctx Context. * @param out Output stream. - * @param schema Schema holder * @return Writer instance. */ - public BinaryWriterEx writer(BinaryContext ctx, BinaryOutputStream out, BinaryWriterSchemaHolder schema); + public BinaryWriterEx writerWithoutSchema(BinaryContext ctx, BinaryOutputStream out); /** * Create accessor for the field. diff --git a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index e10222a6593cd..45146bb29d2fc 100644 --- a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -2897,8 +2897,8 @@ public static BinaryWriterEx writer(BinaryContext ctx, BinaryOutputStream out) { * @param out Output stream. * @return Writer instance. */ - public static BinaryWriterEx writer(BinaryContext ctx, BinaryOutputStream out, BinaryWriterSchemaHolder schema) { - return binariesFactory.writer(ctx, out, schema); + public static BinaryWriterEx writerWithoutSchema(BinaryContext ctx, BinaryOutputStream out) { + return binariesFactory.writerWithoutSchema(ctx, out); } /** @return Instance of caching handler. */ diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinariesFactoryImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinariesFactoryImpl.java index 0064ab7cb6e30..211eb67c17e8e 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinariesFactoryImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinariesFactoryImpl.java @@ -90,8 +90,8 @@ public class BinariesFactoryImpl implements BinariesFactory { } /** {@inheritDoc} */ - @Override public BinaryWriterEx writer(BinaryContext ctx, BinaryOutputStream out, BinaryWriterSchemaHolder schema) { - return new BinaryWriterExImpl(ctx, out, schema, null, false, GridBinaryMarshaller.UNREGISTERED_TYPE_ID); + @Override public BinaryWriterEx writerWithoutSchema(BinaryContext ctx, BinaryOutputStream out) { + return new BinaryWriterExImpl(ctx, out, null, null, false, GridBinaryMarshaller.UNREGISTERED_TYPE_ID); } /** {@inheritDoc} */ diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java index 033be969abcf7..176a1ec9eb155 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java @@ -60,7 +60,7 @@ class BinaryWriterExImpl implements BinaryWriterEx { private final BinaryOutputStream out; /** Schema. */ - private final BinaryWriterSchemaHolder schema; + private final @Nullable BinaryWriterSchemaHolder schema; /** */ private int typeId; @@ -96,7 +96,7 @@ class BinaryWriterExImpl implements BinaryWriterEx { public BinaryWriterExImpl( BinaryContext ctx, BinaryOutputStream out, - BinaryWriterSchemaHolder schema, + @Nullable BinaryWriterSchemaHolder schema, BinaryWriterHandles handles, boolean failIfUnregistered, int typeId diff --git a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryWriterSchemaHolder.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterSchemaHolder.java similarity index 100% rename from modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryWriterSchemaHolder.java rename to modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterSchemaHolder.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java index 4a3bf02a87c71..faa6d7a5c4c0a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinTcpIo.java @@ -275,7 +275,7 @@ private HandshakeResult handshake(ClientListenerProtocolVersion ver) throws IOEx marsh.setContext(new MarshallerContextImpl(null)); - BinaryWriterEx writer = BinaryUtils.writer(U.binaryContext(marsh), BinaryStreams.outputStream(HANDSHAKE_MSG_SIZE), null); + BinaryWriterEx writer = BinaryUtils.writerWithoutSchema(U.binaryContext(marsh), BinaryStreams.outputStream(HANDSHAKE_MSG_SIZE)); writer.writeByte((byte)ClientListenerRequest.HANDSHAKE); @@ -436,8 +436,7 @@ else if (VER_2_1_0.equals(srvProtoVer0)) * @throws SQLException On connection reject. */ private HandshakeResult handshake_2_1_0() throws IOException, SQLException { - BinaryWriterEx writer = BinaryUtils.writer(null, BinaryStreams.outputStream(HANDSHAKE_MSG_SIZE), - null); + BinaryWriterEx writer = BinaryUtils.writerWithoutSchema(null, BinaryStreams.outputStream(HANDSHAKE_MSG_SIZE)); writer.writeByte((byte)ClientListenerRequest.HANDSHAKE); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java index 2f9382a44fde7..b72125fad2625 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java @@ -374,7 +374,7 @@ private void onHandshake(GridNioSession ses, ClientMessage msg) { ClientListenerProtocolVersion ver = ClientListenerProtocolVersion.create(verMajor, verMinor, verMaintenance); - BinaryWriterEx writer = BinaryUtils.writer(null, BinaryStreams.outputStream(8), null); + BinaryWriterEx writer = BinaryUtils.writerWithoutSchema(null, BinaryStreams.outputStream(8)); byte clientType = reader.readByte(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/client/thin/events/FakeIgniteServer.java b/modules/core/src/test/java/org/apache/ignite/internal/client/thin/events/FakeIgniteServer.java index 6288699c7b2d3..db6b74e69bc4a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/client/thin/events/FakeIgniteServer.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/client/thin/events/FakeIgniteServer.java @@ -224,7 +224,7 @@ public void stop() { /** */ private ByteBuffer createMessage(Consumer writerAction) { - try (BinaryWriterEx writer = BinaryUtils.writer(null, BinaryStreams.outputStream(32), null)) { + try (BinaryWriterEx writer = BinaryUtils.writerWithoutSchema(null, BinaryStreams.outputStream(32))) { writer.writeInt(0); writerAction.accept(writer); diff --git a/modules/core/src/test/java/org/apache/ignite/platform/PlatformGetInternalCachesTask.java b/modules/core/src/test/java/org/apache/ignite/platform/PlatformGetInternalCachesTask.java index ee76745387351..af4d47d929645 100644 --- a/modules/core/src/test/java/org/apache/ignite/platform/PlatformGetInternalCachesTask.java +++ b/modules/core/src/test/java/org/apache/ignite/platform/PlatformGetInternalCachesTask.java @@ -68,7 +68,7 @@ private static class InternalCachesJob extends ComputeJobAdapter { @Override public byte[] execute() { IgniteEx ign = (IgniteEx)ignite; - try (BinaryWriterEx writer = BinaryUtils.writer(U.binaryContext(null), BinaryStreams.outputStream(512), null)) { + try (BinaryWriterEx writer = BinaryUtils.writerWithoutSchema(U.binaryContext(null), BinaryStreams.outputStream(512))) { Collection> caches = ign.cachesx(); writer.writeInt(caches.size()); diff --git a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientAtomicLongImpl.java b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientAtomicLongImpl.java index 29055ea3d3fc3..17e2df8063328 100644 --- a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientAtomicLongImpl.java +++ b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientAtomicLongImpl.java @@ -134,7 +134,7 @@ public ClientAtomicLongImpl(String name, @Nullable String groupName, ReliableCha * @param out Output channel. */ private void writeName(PayloadOutputChannel out) { - try (BinaryWriterEx w = BinaryUtils.writer(null, out.out(), null)) { + try (BinaryWriterEx w = BinaryUtils.writerWithoutSchema(null, out.out())) { w.writeString(name); w.writeString(groupName); } diff --git a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientCacheAffinityContext.java b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientCacheAffinityContext.java index fb3877dbdee6d..322e6227b9685 100644 --- a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientCacheAffinityContext.java +++ b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientCacheAffinityContext.java @@ -250,7 +250,7 @@ else if (newMapping.topologyVersion().equals(oldMapping.topologyVersion())) * @param ch Payload output channel. */ public void writeDataCenterNodesRequest(PayloadOutputChannel ch) { - try (BinaryWriterEx w = BinaryUtils.writer(null, ch.out(), null)) { + try (BinaryWriterEx w = BinaryUtils.writerWithoutSchema(null, ch.out())) { w.writeString(dataCenterId); } } diff --git a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientCacheAffinityMapping.java b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientCacheAffinityMapping.java index 1ee7b28fb7eef..c967987809ace 100644 --- a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientCacheAffinityMapping.java +++ b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientCacheAffinityMapping.java @@ -173,7 +173,7 @@ public static void writeRequest( out.writeBoolean(customMappingsRequired); if (ctx.isFeatureSupported(DC_AWARE)) { - try (BinaryWriterEx w = BinaryUtils.writer(null, out, null)) { + try (BinaryWriterEx w = BinaryUtils.writerWithoutSchema(null, out)) { w.writeString(dcId); } } diff --git a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientUtils.java b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientUtils.java index 0bf8047dd3a1c..c6d720c5565ce 100644 --- a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientUtils.java +++ b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/ClientUtils.java @@ -224,7 +224,7 @@ BinaryMetadata binaryMetadata(BinaryInputStream in) throws IOException { /** Serialize binary type metadata to stream. */ void binaryMetadata(BinaryMetadata meta, BinaryOutputStream out) { - try (BinaryWriterEx w = BinaryUtils.writer(marsh.context(), out, null)) { + try (BinaryWriterEx w = BinaryUtils.writerWithoutSchema(marsh.context(), out)) { w.writeInt(meta.typeId()); w.writeString(meta.typeName()); w.writeString(meta.affinityKeyFieldName()); @@ -269,7 +269,7 @@ void binaryMetadata(BinaryMetadata meta, BinaryOutputStream out) { /** Serialize configuration to stream. */ void cacheConfiguration(ClientCacheConfiguration cfg, boolean sql, BinaryOutputStream out, ProtocolContext protocolCtx) { - try (BinaryWriterEx writer = BinaryUtils.writer(marsh.context(), out, null)) { + try (BinaryWriterEx writer = BinaryUtils.writerWithoutSchema(marsh.context(), out)) { if (protocolCtx.isFeatureSupported(ProtocolBitmaskFeature.SQL_CACHE_CREATION)) out.writeBoolean(sql); diff --git a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpClientCache.java b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpClientCache.java index ab574b9cf3eb4..ad634260e35c3 100644 --- a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpClientCache.java +++ b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpClientCache.java @@ -1300,7 +1300,7 @@ private QueryCursor> indexQuery(IndexQuery qry) { BinaryOutputStream out = payloadCh.out(); - try (BinaryWriterEx w = BinaryUtils.writer(marsh.context(), out, null)) { + try (BinaryWriterEx w = BinaryUtils.writerWithoutSchema(marsh.context(), out)) { w.writeInt(qry.getPageSize()); w.writeBoolean(qry.isLocal()); w.writeInt(qry.getPartition() == null ? -1 : qry.getPartition()); diff --git a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java index a7109e0e02ea2..a90fbd3437b4a 100644 --- a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java +++ b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java @@ -845,7 +845,7 @@ else if (!supportedVers.contains(srvVer) || /** Send handshake request. */ private void handshakeReq(ProtocolVersion proposedVer, String user, String pwd, Map userAttrs) throws ClientConnectionException { - try (BinaryWriterEx writer = BinaryUtils.writer(BinaryUtils.binaryContext(null), BinaryStreams.outputStream(32), null)) { + try (BinaryWriterEx writer = BinaryUtils.writerWithoutSchema(BinaryUtils.binaryContext(null), BinaryStreams.outputStream(32))) { ProtocolContext protocolCtx = protocolContextFromVersion(proposedVer); writer.writeInt(0); // reserve an integer for the request size diff --git a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpClientTransactions.java b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpClientTransactions.java index 8dee96182ab78..4030782258b9c 100644 --- a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpClientTransactions.java +++ b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpClientTransactions.java @@ -101,7 +101,7 @@ private ClientTransaction txStart0(TransactionConcurrency concurrency, Transacti protocolCtx.version(), TRANSACTIONS.verIntroduced())); } - try (BinaryWriterEx writer = BinaryUtils.writer(marsh.context(), req.out(), null)) { + try (BinaryWriterEx writer = BinaryUtils.writerWithoutSchema(marsh.context(), req.out())) { writer.writeByte((byte)(concurrency == null ? txCfg.getDefaultTxConcurrency() : concurrency).ordinal()); writer.writeByte((byte)(isolation == null ? txCfg.getDefaultTxIsolation() : isolation).ordinal()); writer.writeLong(timeout == null ? txCfg.getDefaultTxTimeout() : timeout); diff --git a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java index 2db974818cc63..94a49d930a6d5 100644 --- a/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java +++ b/modules/thin-client/impl/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java @@ -555,7 +555,7 @@ private static void ensureCacheConfiguration(ClientCacheConfiguration cfg) { /** Serialize string. */ private void writeString(String s, BinaryOutputStream out) { - try (BinaryWriterEx w = BinaryUtils.writer(marsh.context(), out, null)) { + try (BinaryWriterEx w = BinaryUtils.writerWithoutSchema(marsh.context(), out)) { w.writeString(s); } } From 5587e948e7a6b9fee564cde4ef20cb635deea6d8 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 9 Sep 2026 12:49:38 +0300 Subject: [PATCH 14/20] IGNITE-29044 Move BinaryWriterSchemaHolder to binary-impl --- .../internal/binary/BinaryWriterExImpl.java | 18 +++++++++--------- .../binary/BinaryWriterSchemaHolder.java | 10 +++++----- .../builder/BinaryObjectBuilderImpl.java | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java index 176a1ec9eb155..8eb2efe962281 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java @@ -235,11 +235,11 @@ int position() { if (userType) { if (ctx.isCompactFooter()) { - flags = BinaryUtils.FLAG_USR_TYP | BinaryUtils.FLAG_COMPACT_FOOTER; + flags = BinaryImplUtils.FLAG_USR_TYP | BinaryImplUtils.FLAG_COMPACT_FOOTER; useCompactFooter = true; } else { - flags = BinaryUtils.FLAG_USR_TYP; + flags = BinaryImplUtils.FLAG_USR_TYP; useCompactFooter = false; } } @@ -254,18 +254,18 @@ int position() { offset = out.position() - start; // Write the schema. - flags |= BinaryUtils.FLAG_HAS_SCHEMA; + flags |= BinaryImplUtils.FLAG_HAS_SCHEMA; int offsetByteCnt = schema.write(out, fieldCnt, useCompactFooter); - if (offsetByteCnt == BinaryUtils.OFFSET_1) - flags |= BinaryUtils.FLAG_OFFSET_ONE_BYTE; - else if (offsetByteCnt == BinaryUtils.OFFSET_2) - flags |= BinaryUtils.FLAG_OFFSET_TWO_BYTES; + if (offsetByteCnt == BinaryImplUtils.OFFSET_1) + flags |= BinaryImplUtils.FLAG_OFFSET_ONE_BYTE; + else if (offsetByteCnt == BinaryImplUtils.OFFSET_2) + flags |= BinaryImplUtils.FLAG_OFFSET_TWO_BYTES; // Write raw offset if needed. if (rawOffPos != 0) { - flags |= BinaryUtils.FLAG_HAS_RAW; + flags |= BinaryImplUtils.FLAG_HAS_RAW; out.writeInt(rawOffPos - start); } @@ -275,7 +275,7 @@ else if (offsetByteCnt == BinaryUtils.OFFSET_2) offset = rawOffPos - start; // If there is no schema, we are free to write raw offset to schema offset. - flags |= BinaryUtils.FLAG_HAS_RAW; + flags |= BinaryImplUtils.FLAG_HAS_RAW; } else offset = GridBinaryMarshaller.DFLT_HDR_LEN; diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterSchemaHolder.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterSchemaHolder.java index 8e5eab4f9f6ee..fecb10aa23927 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterSchemaHolder.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterSchemaHolder.java @@ -96,19 +96,19 @@ public int write(BinaryOutputStream out, int fieldCnt, boolean compactFooter) { for (int curIdx = startIdx + 1; curIdx < idx; curIdx += 2) out.unsafeWriteByte((byte)data[curIdx]); - res = BinaryUtils.OFFSET_1; + res = BinaryImplUtils.OFFSET_1; } else if (lastOffset < MAX_OFFSET_2) { for (int curIdx = startIdx + 1; curIdx < idx; curIdx += 2) out.unsafeWriteShort((short)data[curIdx]); - res = BinaryUtils.OFFSET_2; + res = BinaryImplUtils.OFFSET_2; } else { for (int curIdx = startIdx + 1; curIdx < idx; curIdx += 2) out.unsafeWriteInt(data[curIdx]); - res = BinaryUtils.OFFSET_4; + res = BinaryImplUtils.OFFSET_4; } } else { @@ -118,7 +118,7 @@ else if (lastOffset < MAX_OFFSET_2) { out.unsafeWriteByte((byte)data[curIdx++]); } - res = BinaryUtils.OFFSET_1; + res = BinaryImplUtils.OFFSET_1; } else if (lastOffset < MAX_OFFSET_2) { for (int curIdx = startIdx; curIdx < idx;) { @@ -126,7 +126,7 @@ else if (lastOffset < MAX_OFFSET_2) { out.unsafeWriteShort((short)data[curIdx++]); } - res = BinaryUtils.OFFSET_2; + res = BinaryImplUtils.OFFSET_2; } else { for (int curIdx = startIdx; curIdx < idx;) { diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java index 9a19d3dcb994c..d1ef8426cc90a 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java @@ -208,7 +208,7 @@ void serializeTo(BinaryWriterEx writer, BinaryBuilderSerializer serializer) { Map fieldsMeta = null; - if (reader != null && BinaryUtils.hasSchema(flags)) { + if (reader != null && BinaryImplUtils.hasSchema(flags)) { Map assignedFldsById; if (assignedVals != null) { From 788e0ed0781f1f1821eefa659c9d38ac6e602798 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 9 Sep 2026 13:48:23 +0300 Subject: [PATCH 15/20] IGNITE-29044 Move BinaryWriterSchemaHolder to binary-impl --- .../org/apache/ignite/internal/binary/BinaryObjectEx.java | 5 +++++ .../org/apache/ignite/internal/binary/BinaryObjectImpl.java | 6 ++++++ .../ignite/internal/binary/BinaryObjectOffheapImpl.java | 6 ++++++ .../internal/processors/cache/verify/IdleVerifyUtility.java | 3 +-- .../org/apache/ignite/client/BinaryConfigurationTest.java | 2 +- .../binary/BinarySerialiedFieldComparatorSelfTest.java | 2 +- .../internal/binary/BinarySerializedFieldComparator.java | 4 ++-- 7 files changed, 22 insertions(+), 6 deletions(-) diff --git a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryObjectEx.java b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryObjectEx.java index f8e5c067810b0..f5d8d61d62e47 100644 --- a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryObjectEx.java +++ b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryObjectEx.java @@ -141,4 +141,9 @@ public default void detachAllowed(boolean detachAllowed) { public default boolean hasCircularReferences() { return false; } + + /** @return {@code true} if binary object has compact footer. */ + public default boolean isCompactFooter() { + return false; + } } diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java index a0a9c9c9c90af..d41d6a3160d1a 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java @@ -45,6 +45,7 @@ import org.jetbrains.annotations.Nullable; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.ignite.internal.binary.BinaryImplUtils.FLAG_COMPACT_FOOTER; import static org.apache.ignite.internal.binary.GridBinaryMarshaller.TRANSFORMED; /** @@ -348,6 +349,11 @@ public void context(BinaryContext ctx) { return BinaryImplUtils.isFlagSet(flags, flag); } + /** {@inheritDoc} */ + @Override public boolean isCompactFooter() { + return isFlagSet(FLAG_COMPACT_FOOTER); + } + /** {@inheritDoc} */ @Override public int typeId() { int off = start + GridBinaryMarshaller.TYPE_ID_POS; diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java index 665607bc109a0..9ad6c06a605d4 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryObjectOffheapImpl.java @@ -43,6 +43,7 @@ import org.jetbrains.annotations.Nullable; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.ignite.internal.binary.BinaryImplUtils.FLAG_COMPACT_FOOTER; /** * Binary object implementation over offheap memory @@ -178,6 +179,11 @@ public BinaryObjectOffheapImpl(BinaryContext ctx, long ptr, int start, int size) return BinaryImplUtils.isFlagSet(flags, flag); } + /** {@inheritDoc} */ + @Override public boolean isCompactFooter() { + return isFlagSet(FLAG_COMPACT_FOOTER); + } + /** {@inheritDoc} */ @Nullable @Override public BinaryType type() throws BinaryObjectException { return BinaryUtils.typeProxy(ctx, this); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java index 9e9b2b6a90ccb..3cccbb3e72a3a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyUtility.java @@ -55,7 +55,6 @@ import org.apache.ignite.lang.IgniteInClosure; import org.jetbrains.annotations.Nullable; -import static org.apache.ignite.internal.binary.BinaryUtils.FLAG_COMPACT_FOOTER; import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_AUX; import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX; @@ -421,7 +420,7 @@ public void update( if (key.cacheObjectType() == TYPE_BINARY) { binary++; - if (((BinaryObjectEx)key).isFlagSet(FLAG_COMPACT_FOOTER)) + if (((BinaryObjectEx)key).isCompactFooter()) cf++; else noCf++; diff --git a/modules/core/src/test/java/org/apache/ignite/client/BinaryConfigurationTest.java b/modules/core/src/test/java/org/apache/ignite/client/BinaryConfigurationTest.java index 543e020ccddf2..965616d14db6d 100644 --- a/modules/core/src/test/java/org/apache/ignite/client/BinaryConfigurationTest.java +++ b/modules/core/src/test/java/org/apache/ignite/client/BinaryConfigurationTest.java @@ -31,7 +31,7 @@ import org.apache.ignite.testframework.LogListener; import org.junit.Test; -import static org.apache.ignite.internal.binary.BinaryUtils.FLAG_COMPACT_FOOTER; +import static org.apache.ignite.internal.binary.BinaryImplUtils.FLAG_COMPACT_FOOTER; /** * Tests binary configuration behavior. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerialiedFieldComparatorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerialiedFieldComparatorSelfTest.java index 868e35223e4dd..1fafa4eaa3fb3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerialiedFieldComparatorSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerialiedFieldComparatorSelfTest.java @@ -503,7 +503,7 @@ private BinarySerializedFieldComparator comparator(BinaryObjectExImpl obj) { ? BinaryPrimitives.readShort(ptr, start + GridBinaryMarshaller.FLAGS_POS) : BinaryPrimitives.readShort(arr, start + GridBinaryMarshaller.FLAGS_POS); - int fieldIdLen = BinaryUtils.isCompactFooter(flags) ? 0 : BinaryUtils.FIELD_ID_LEN; + int fieldIdLen = BinaryImplUtils.isCompactFooter(flags) ? 0 : BinaryImplUtils.FIELD_ID_LEN; int fieldOffLen = BinaryImplUtils.fieldOffsetLength(flags); int orderBase = start + schemaOff + fieldIdLen; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java index 150c1475bda26..5768886f2a3f9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java @@ -94,12 +94,12 @@ public void findField(int order) { else { int pos = orderBase + order * orderMultiplier; - if (fieldOffLen == BinaryUtils.OFFSET_1) { + if (fieldOffLen == BinaryImplUtils.OFFSET_1) { byte val = offheap() ? BinaryPrimitives.readByte(ptr, pos) : BinaryPrimitives.readByte(arr, pos); curFieldPos = startOff + ((int)val & 0xFF); } - else if (fieldOffLen == BinaryUtils.OFFSET_2) { + else if (fieldOffLen == BinaryImplUtils.OFFSET_2) { short val = offheap() ? BinaryPrimitives.readShort(ptr, pos) : BinaryPrimitives.readShort(arr, pos); curFieldPos = startOff + ((int)val & 0xFFFF); From 02bb629fea2fa1c58037493827f25a79235de32c Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 9 Sep 2026 18:40:25 +0300 Subject: [PATCH 16/20] IGNITE-29044 Move BinaryWriterSchemaHolder to binary-impl --- .../ignite/internal/binary/BinaryUtils.java | 268 ++---------------- .../binary/BinaryArrayIdentityResolver.java | 2 +- .../internal/binary/BinaryImplUtils.java | 200 +++++++++++++ .../binary/CrossObjectReferenceResolver.java | 2 +- .../builder/BinaryBuilderSerializer.java | 5 +- .../builder/BinaryObjectBuilderImpl.java | 6 +- .../BinarySerializedFieldComparator.java | 2 +- .../metric/MetricsConfigurationTest.java | 2 +- .../ignite/testframework/GridTestUtils.java | 40 +++ 9 files changed, 267 insertions(+), 260 deletions(-) diff --git a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index c4b1bffabc013..1829370cc4ef2 100644 --- a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -38,7 +38,6 @@ import java.sql.Time; import java.sql.Timestamp; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -331,172 +330,6 @@ public static String fieldTypeName(int typeId) { return FIELD_TYPE_NAMES[typeId]; } - /** - * Write value with flag. e.g. writePlainObject(writer, (byte)77) will write two byte: {BYTE, 77}. - * - * @param writer W - * @param val Value. - */ - public static void writePlainObject(BinaryWriterEx writer, Object val) { - Byte flag = PLAIN_CLASS_TO_FLAG.get(val.getClass()); - - if (flag == null) - throw new IllegalArgumentException("Can't write object with type: " + val.getClass()); - - switch (flag) { - case GridBinaryMarshaller.BYTE: - writer.writeByte(flag); - writer.writeByte((Byte)val); - - break; - - case GridBinaryMarshaller.SHORT: - writer.writeByte(flag); - writer.writeShort((Short)val); - - break; - - case GridBinaryMarshaller.INT: - writer.writeByte(flag); - writer.writeInt((Integer)val); - - break; - - case GridBinaryMarshaller.LONG: - writer.writeByte(flag); - writer.writeLong((Long)val); - - break; - - case GridBinaryMarshaller.FLOAT: - writer.writeByte(flag); - writer.writeFloat((Float)val); - - break; - - case GridBinaryMarshaller.DOUBLE: - writer.writeByte(flag); - writer.writeDouble((Double)val); - - break; - - case GridBinaryMarshaller.CHAR: - writer.writeByte(flag); - writer.writeChar((Character)val); - - break; - - case GridBinaryMarshaller.BOOLEAN: - writer.writeByte(flag); - writer.writeBoolean((Boolean)val); - - break; - - case GridBinaryMarshaller.DECIMAL: - writer.writeDecimal((BigDecimal)val); - - break; - - case GridBinaryMarshaller.STRING: - writer.writeString((String)val); - - break; - - case GridBinaryMarshaller.UUID: - writer.writeUuid((UUID)val); - - break; - - case GridBinaryMarshaller.DATE: - writer.writeDate((Date)val); - - break; - - case GridBinaryMarshaller.TIMESTAMP: - writer.writeTimestamp((Timestamp)val); - - break; - - case GridBinaryMarshaller.TIME: - writer.writeTime((Time)val); - - break; - - case GridBinaryMarshaller.BYTE_ARR: - writer.writeByteArray((byte[])val); - - break; - - case GridBinaryMarshaller.SHORT_ARR: - writer.writeShortArray((short[])val); - - break; - - case GridBinaryMarshaller.INT_ARR: - writer.writeIntArray((int[])val); - - break; - - case GridBinaryMarshaller.LONG_ARR: - writer.writeLongArray((long[])val); - - break; - - case GridBinaryMarshaller.FLOAT_ARR: - writer.writeFloatArray((float[])val); - - break; - - case GridBinaryMarshaller.DOUBLE_ARR: - writer.writeDoubleArray((double[])val); - - break; - - case GridBinaryMarshaller.CHAR_ARR: - writer.writeCharArray((char[])val); - - break; - - case GridBinaryMarshaller.BOOLEAN_ARR: - writer.writeBooleanArray((boolean[])val); - - break; - - case GridBinaryMarshaller.DECIMAL_ARR: - writer.writeDecimalArray((BigDecimal[])val); - - break; - - case GridBinaryMarshaller.STRING_ARR: - writer.writeStringArray((String[])val); - - break; - - case GridBinaryMarshaller.UUID_ARR: - writer.writeUuidArray((UUID[])val); - - break; - - case GridBinaryMarshaller.DATE_ARR: - writer.writeDateArray((Date[])val); - - break; - - case GridBinaryMarshaller.TIMESTAMP_ARR: - writer.writeTimestampArray((Timestamp[])val); - - break; - - case GridBinaryMarshaller.TIME_ARR: - writer.writeTimeArray((Time[])val); - - break; - - default: - throw new IllegalArgumentException("Can't write object with type: " + val.getClass()); - } - } - /** * @param obj Value to unwrap. * @return Unwrapped value. @@ -2535,14 +2368,6 @@ public static Map, Function> unwrapFuncForSizeCalc() { ); } - /** - * @param val Value to check. - * @return {@code True} if {@code val} instance of {@link BinaryEnumArray}. - */ - public static boolean isBinaryEnumArray(Object val) { - return val instanceof BinaryEnumArray; - } - /** * @param val Value to check. * @return {@code True} if {@code val} instance of binary Enum object. @@ -2573,10 +2398,10 @@ public static BinaryReaderEx reader(BinaryContext ctx, BinaryInputStream in, Cla * @param forUnmarshal {@code True} if reader is need to unmarshal object. */ public static BinaryReaderEx reader(BinaryContext ctx, - BinaryInputStream in, - ClassLoader ldr, - BinaryReaderEx reader, - boolean forUnmarshal) { + BinaryInputStream in, + ClassLoader ldr, + BinaryReaderEx reader, + boolean forUnmarshal) { return reader(ctx, in, ldr, reader.handles(), forUnmarshal); } @@ -2590,10 +2415,10 @@ public static BinaryReaderEx reader(BinaryContext ctx, * @param forUnmarshal {@code True} if reader is need to unmarshal object. */ static BinaryReaderEx reader(BinaryContext ctx, - BinaryInputStream in, - ClassLoader ldr, - @Nullable BinaryReaderHandles hnds, - boolean forUnmarshal) { + BinaryInputStream in, + ClassLoader ldr, + @Nullable BinaryReaderHandles hnds, + boolean forUnmarshal) { return binariesFactory.reader(ctx, in, ldr, hnds, forUnmarshal); } @@ -2607,10 +2432,10 @@ static BinaryReaderEx reader(BinaryContext ctx, * @param forUnmarshal {@code True} if reader is need to unmarshal object. */ public static BinaryReaderEx reader(BinaryContext ctx, - BinaryInputStream in, - ClassLoader ldr, - boolean skipHdrCheck, - boolean forUnmarshal) { + BinaryInputStream in, + ClassLoader ldr, + boolean skipHdrCheck, + boolean forUnmarshal) { return reader(ctx, in, ldr, null, skipHdrCheck, forUnmarshal); } @@ -2625,11 +2450,11 @@ public static BinaryReaderEx reader(BinaryContext ctx, * @param forUnmarshal {@code True} if reader is need to unmarshal object. */ static BinaryReaderEx reader(BinaryContext ctx, - BinaryInputStream in, - ClassLoader ldr, - @Nullable BinaryReaderHandles hnds, - boolean skipHdrCheck, - boolean forUnmarshal) { + BinaryInputStream in, + ClassLoader ldr, + @Nullable BinaryReaderHandles hnds, + boolean skipHdrCheck, + boolean forUnmarshal) { return binariesFactory.reader(ctx, in, ldr, hnds, skipHdrCheck, forUnmarshal); } @@ -2721,45 +2546,6 @@ public static Object field(Object obj, int fieldId) { return ((BinaryObjectExImpl)obj).field(fieldId); } - /** - * Check for arrays equality. - * - * @param a1 Value 1. - * @param a2 Value 2. - * @return {@code True} if arrays equal. - */ - public static boolean arrayEq(Object a1, Object a2) { - if (a1 == a2) - return true; - - if (a1 == null || a2 == null) - return a1 != null || a2 != null; - - if (a1.getClass() != a2.getClass()) - return false; - - if (a1 instanceof byte[]) - return Arrays.equals((byte[])a1, (byte[])a2); - else if (a1 instanceof boolean[]) - return Arrays.equals((boolean[])a1, (boolean[])a2); - else if (a1 instanceof short[]) - return Arrays.equals((short[])a1, (short[])a2); - else if (a1 instanceof char[]) - return Arrays.equals((char[])a1, (char[])a2); - else if (a1 instanceof int[]) - return Arrays.equals((int[])a1, (int[])a2); - else if (a1 instanceof long[]) - return Arrays.equals((long[])a1, (long[])a2); - else if (a1 instanceof float[]) - return Arrays.equals((float[])a1, (float[])a2); - else if (a1 instanceof double[]) - return Arrays.equals((double[])a1, (double[])a2); - else if (isBinaryArray(a1)) - return a1.equals(a2); - - return Arrays.deepEquals((Object[])a1, (Object[])a2); - } - /** * @param o Object to detach. * @return Detached object. @@ -2785,16 +2571,6 @@ public static Collection> schemasAndFieldsIds(BinaryMetadata return F.viewReadOnly(meta.schemas(), s -> new T2<>(s.schemaId(), s.fieldIds())); } - /** - * Gets field by its order. - * - * @param reader Reader. - * @param order Order. - */ - public static int fieldId(BinaryReaderEx reader, int order) { - return reader.getOrCreateSchema().fieldId(order); - } - /** * @param obj Value. * @return {@code True} if object is of known immutable type. @@ -2830,16 +2606,6 @@ public static BinaryMetadata binaryMetadata( return new BinaryMetadata(typeId, typeName, fields, affKeyFieldName, schemas, isEnum, enumMap); } - /** */ - public static int hashCode(byte[] data, int startPos, int endPos) { - int hash = 1; - - for (int i = startPos; i < endPos; i++) - hash = 31 * hash + data[i]; - - return hash; - } - /** * Enum type. */ diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryArrayIdentityResolver.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryArrayIdentityResolver.java index 86109b19b6264..3603a98c7bbe1 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryArrayIdentityResolver.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryArrayIdentityResolver.java @@ -57,7 +57,7 @@ private BinaryArrayIdentityResolver() { int end = ex.footerStartOffset(); if (ex.hasBytes()) - return BinaryUtils.hashCode(ex.bytes(), start, end); + return BinaryImplUtils.hashCode(ex.bytes(), start, end); else { // Handle offheap object. int hash = 1; diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryImplUtils.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryImplUtils.java index 2e23d5ca265dc..73769935858e8 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryImplUtils.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryImplUtils.java @@ -16,9 +16,15 @@ */ package org.apache.ignite.internal.binary; +import java.math.BigDecimal; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Date; +import java.util.UUID; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.lang.IgniteBiTuple; +import static org.apache.ignite.internal.binary.BinaryUtils.PLAIN_CLASS_TO_FLAG; import static org.apache.ignite.internal.binary.BinaryUtils.length; /** @@ -299,4 +305,198 @@ public static boolean isPlainArrayType(int type) { return (type >= GridBinaryMarshaller.BYTE_ARR && type <= GridBinaryMarshaller.DATE_ARR) || type == GridBinaryMarshaller.TIMESTAMP_ARR || type == GridBinaryMarshaller.TIME_ARR; } + + /** + * Write value with flag. e.g. writePlainObject(writer, (byte)77) will write two byte: {BYTE, 77}. + * + * @param writer W + * @param val Value. + */ + public static void writePlainObject(BinaryWriterEx writer, Object val) { + Byte flag = PLAIN_CLASS_TO_FLAG.get(val.getClass()); + + if (flag == null) + throw new IllegalArgumentException("Can't write object with type: " + val.getClass()); + + switch (flag) { + case GridBinaryMarshaller.BYTE: + writer.writeByte(flag); + writer.writeByte((Byte)val); + + break; + + case GridBinaryMarshaller.SHORT: + writer.writeByte(flag); + writer.writeShort((Short)val); + + break; + + case GridBinaryMarshaller.INT: + writer.writeByte(flag); + writer.writeInt((Integer)val); + + break; + + case GridBinaryMarshaller.LONG: + writer.writeByte(flag); + writer.writeLong((Long)val); + + break; + + case GridBinaryMarshaller.FLOAT: + writer.writeByte(flag); + writer.writeFloat((Float)val); + + break; + + case GridBinaryMarshaller.DOUBLE: + writer.writeByte(flag); + writer.writeDouble((Double)val); + + break; + + case GridBinaryMarshaller.CHAR: + writer.writeByte(flag); + writer.writeChar((Character)val); + + break; + + case GridBinaryMarshaller.BOOLEAN: + writer.writeByte(flag); + writer.writeBoolean((Boolean)val); + + break; + + case GridBinaryMarshaller.DECIMAL: + writer.writeDecimal((BigDecimal)val); + + break; + + case GridBinaryMarshaller.STRING: + writer.writeString((String)val); + + break; + + case GridBinaryMarshaller.UUID: + writer.writeUuid((UUID)val); + + break; + + case GridBinaryMarshaller.DATE: + writer.writeDate((Date)val); + + break; + + case GridBinaryMarshaller.TIMESTAMP: + writer.writeTimestamp((Timestamp)val); + + break; + + case GridBinaryMarshaller.TIME: + writer.writeTime((Time)val); + + break; + + case GridBinaryMarshaller.BYTE_ARR: + writer.writeByteArray((byte[])val); + + break; + + case GridBinaryMarshaller.SHORT_ARR: + writer.writeShortArray((short[])val); + + break; + + case GridBinaryMarshaller.INT_ARR: + writer.writeIntArray((int[])val); + + break; + + case GridBinaryMarshaller.LONG_ARR: + writer.writeLongArray((long[])val); + + break; + + case GridBinaryMarshaller.FLOAT_ARR: + writer.writeFloatArray((float[])val); + + break; + + case GridBinaryMarshaller.DOUBLE_ARR: + writer.writeDoubleArray((double[])val); + + break; + + case GridBinaryMarshaller.CHAR_ARR: + writer.writeCharArray((char[])val); + + break; + + case GridBinaryMarshaller.BOOLEAN_ARR: + writer.writeBooleanArray((boolean[])val); + + break; + + case GridBinaryMarshaller.DECIMAL_ARR: + writer.writeDecimalArray((BigDecimal[])val); + + break; + + case GridBinaryMarshaller.STRING_ARR: + writer.writeStringArray((String[])val); + + break; + + case GridBinaryMarshaller.UUID_ARR: + writer.writeUuidArray((UUID[])val); + + break; + + case GridBinaryMarshaller.DATE_ARR: + writer.writeDateArray((Date[])val); + + break; + + case GridBinaryMarshaller.TIMESTAMP_ARR: + writer.writeTimestampArray((Timestamp[])val); + + break; + + case GridBinaryMarshaller.TIME_ARR: + writer.writeTimeArray((Time[])val); + + break; + + default: + throw new IllegalArgumentException("Can't write object with type: " + val.getClass()); + } + } + + /** + * @param val Value to check. + * @return {@code True} if {@code val} instance of {@link BinaryEnumArray}. + */ + public static boolean isBinaryEnumArray(Object val) { + return val instanceof BinaryEnumArray; + } + + /** + * Gets field by its order. + * + * @param reader Reader. + * @param order Order. + */ + public static int fieldId(BinaryReaderEx reader, int order) { + return reader.getOrCreateSchema().fieldId(order); + } + + /** */ + public static int hashCode(byte[] data, int startPos, int endPos) { + int hash = 1; + + for (int i = startPos; i < endPos; i++) + hash = 31 * hash + data[i]; + + return hash; + } } diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/CrossObjectReferenceResolver.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/CrossObjectReferenceResolver.java index 42815f6b8f3b0..d5c06d7032e44 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/CrossObjectReferenceResolver.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/CrossObjectReferenceResolver.java @@ -188,7 +188,7 @@ private void doObjectProcessing() { overrideHeader( outObjStartPos, /** flags */ setFieldOffsetFlag(inObjDesc.flags, footerFieldOffsetLen), - /** hash */ BinaryUtils.hashCode(out.array(), outObjStartPos + DFLT_HDR_LEN, outFooterStartPos), + /** hash */ BinaryImplUtils.hashCode(out.array(), outObjStartPos + DFLT_HDR_LEN, outFooterStartPos), /** total length */ out.position() - outObjStartPos, schemaOrRawOffsetPos ); diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java index c30f1daf9a915..f2e1926a30d1a 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java @@ -22,6 +22,7 @@ import java.util.Map; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.internal.binary.BinariesFactoryImpl; +import org.apache.ignite.internal.binary.BinaryImplUtils; import org.apache.ignite.internal.binary.BinaryObjectEx; import org.apache.ignite.internal.binary.BinaryUtils; import org.apache.ignite.internal.binary.BinaryWriterEx; @@ -174,12 +175,12 @@ public void writeValue(BinaryWriterEx writer, Object val, boolean forceCol, bool Byte flag = BinaryUtils.PLAIN_CLASS_TO_FLAG.get(val.getClass()); if (flag != null) { - BinaryUtils.writePlainObject(writer, val); + BinaryImplUtils.writePlainObject(writer, val); return; } - if (BinaryUtils.isBinaryEnumArray(val)) { + if (BinaryImplUtils.isBinaryEnumArray(val)) { BinaryObjectEx val0 = (BinaryObjectEx)val; if (val0.componentTypeId() == GridBinaryMarshaller.UNREGISTERED_TYPE_ID) diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java index d1ef8426cc90a..33d372f374181 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java @@ -249,7 +249,7 @@ void serializeTo(BinaryWriterEx writer, BinaryBuilderSerializer serializer) { int idx = 0; while (reader.position() < rawPos) { - int fieldId = BinaryUtils.fieldId(reader.reader(), idx++); + int fieldId = BinaryImplUtils.fieldId(reader.reader(), idx++); int fieldLen = fieldPositionAndLength(footerPos, footerEnd, rawPos, fieldIdLen, fieldOffsetLen).get2(); @@ -382,7 +382,7 @@ else if (newVal.getClass().isArray() && else if (newVal.getClass().isArray() && BinaryObject.class.isAssignableFrom(newVal.getClass().getComponentType())) newFldTypeId = GridBinaryMarshaller.OBJ_ARR; - else if (BinaryUtils.isBinaryEnumArray(newVal)) + else if (BinaryImplUtils.isBinaryEnumArray(newVal)) newFldTypeId = GridBinaryMarshaller.ENUM_ARR; else if (BinaryUtils.isBinaryArray(newVal)) @@ -475,7 +475,7 @@ private void ensureReadCacheInit() { int idx = 0; while (footerPos + fieldIdLen < footerEnd) { - int fieldId = BinaryUtils.fieldId(reader.reader(), idx++); + int fieldId = BinaryImplUtils.fieldId(reader.reader(), idx++); IgniteBiTuple posAndLen = fieldPositionAndLength(footerPos, footerEnd, rawPos, fieldIdLen, fieldOffsetLen); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java index 5768886f2a3f9..b3421ba9416e0 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinarySerializedFieldComparator.java @@ -21,7 +21,7 @@ import org.apache.ignite.internal.util.offheap.unsafe.GridUnsafeMemory; import org.apache.ignite.internal.util.typedef.F; -import static org.apache.ignite.internal.binary.BinaryUtils.arrayEq; +import static org.apache.ignite.testframework.GridTestUtils.arrayEq; /** * Compares fields in serialized form when possible. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/metric/MetricsConfigurationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/metric/MetricsConfigurationTest.java index 704907816c32a..9b4d472b58f0b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/metric/MetricsConfigurationTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/metric/MetricsConfigurationTest.java @@ -37,7 +37,6 @@ import org.junit.Test; import static org.apache.ignite.configuration.WALMode.FSYNC; -import static org.apache.ignite.internal.binary.BinaryUtils.arrayEq; import static org.apache.ignite.internal.processors.cache.transactions.TransactionMetricsAdapter.METRIC_SYSTEM_TIME_HISTOGRAM; import static org.apache.ignite.internal.processors.metric.GridMetricManager.HISTOGRAM_CFG_PREFIX; import static org.apache.ignite.internal.processors.metric.GridMetricManager.HITRATE_CFG_PREFIX; @@ -47,6 +46,7 @@ import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.metricName; import static org.apache.ignite.internal.processors.pool.PoolProcessor.TASK_EXEC_TIME; import static org.apache.ignite.internal.processors.pool.PoolProcessor.THREAD_POOLS; +import static org.apache.ignite.testframework.GridTestUtils.arrayEq; import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertNotEquals; diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java index 02e9a4d21d250..6a8736060caf4 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java @@ -144,6 +144,7 @@ import static java.lang.Long.parseLong; import static java.util.Comparator.comparingLong; import static org.apache.ignite.IgniteSystemProperties.IGNITE_HOME; +import static org.apache.ignite.internal.binary.BinaryUtils.isBinaryArray; import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION; import static org.apache.ignite.internal.processors.cache.persistence.filename.NodeFileTree.partitionFileName; import static org.apache.ignite.internal.util.lang.ClusterNodeFunc.nodeIds; @@ -2751,4 +2752,43 @@ public static long sizeOfDirectory(File dir) throws IOException { return size[0]; } + + /** + * Check for arrays equality. + * + * @param a1 Value 1. + * @param a2 Value 2. + * @return {@code True} if arrays equal. + */ + public static boolean arrayEq(Object a1, Object a2) { + if (a1 == a2) + return true; + + if (a1 == null || a2 == null) + return a1 != null || a2 != null; + + if (a1.getClass() != a2.getClass()) + return false; + + if (a1 instanceof byte[]) + return Arrays.equals((byte[])a1, (byte[])a2); + else if (a1 instanceof boolean[]) + return Arrays.equals((boolean[])a1, (boolean[])a2); + else if (a1 instanceof short[]) + return Arrays.equals((short[])a1, (short[])a2); + else if (a1 instanceof char[]) + return Arrays.equals((char[])a1, (char[])a2); + else if (a1 instanceof int[]) + return Arrays.equals((int[])a1, (int[])a2); + else if (a1 instanceof long[]) + return Arrays.equals((long[])a1, (long[])a2); + else if (a1 instanceof float[]) + return Arrays.equals((float[])a1, (float[])a2); + else if (a1 instanceof double[]) + return Arrays.equals((double[])a1, (double[])a2); + else if (isBinaryArray(a1)) + return a1.equals(a2); + + return Arrays.deepEquals((Object[])a1, (Object[])a2); + } } From 7eae07c49c38932fe30440fdfd05668ebeca5ff1 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 9 Sep 2026 19:01:34 +0300 Subject: [PATCH 17/20] IGNITE-29044 Move BinaryWriterSchemaHolder to binary-impl --- .../ignite/internal/binary/BinaryUtils.java | 17 ----------------- .../internal/binary/BinaryImplUtils.java | 18 ++++++++++++++++++ .../binary/builder/BinaryBuilderReader.java | 3 ++- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index 1829370cc4ef2..245ec20f67d11 100644 --- a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -2388,23 +2388,6 @@ public static BinaryReaderEx reader(BinaryContext ctx, BinaryInputStream in, Cla return binariesFactory.reader(ctx, in, ldr, forUnmarshal); } - /** - * Creates reader instance. - * - * @param ctx Context. - * @param in Input stream. - * @param ldr Class loader. - * @param reader BinaryReaderEx. - * @param forUnmarshal {@code True} if reader is need to unmarshal object. - */ - public static BinaryReaderEx reader(BinaryContext ctx, - BinaryInputStream in, - ClassLoader ldr, - BinaryReaderEx reader, - boolean forUnmarshal) { - return reader(ctx, in, ldr, reader.handles(), forUnmarshal); - } - /** * Creates reader instance. * diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryImplUtils.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryImplUtils.java index 73769935858e8..6d59f0519b52f 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryImplUtils.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryImplUtils.java @@ -21,6 +21,7 @@ import java.sql.Timestamp; import java.util.Date; import java.util.UUID; +import org.apache.ignite.internal.binary.streams.BinaryInputStream; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.lang.IgniteBiTuple; @@ -499,4 +500,21 @@ public static int hashCode(byte[] data, int startPos, int endPos) { return hash; } + + /** + * Creates reader instance. + * + * @param ctx Context. + * @param in Input stream. + * @param ldr Class loader. + * @param reader BinaryReaderEx. + * @param forUnmarshal {@code True} if reader is need to unmarshal object. + */ + public static BinaryReaderEx reader(BinaryContext ctx, + BinaryInputStream in, + ClassLoader ldr, + BinaryReaderEx reader, + boolean forUnmarshal) { + return BinaryUtils.binariesFactory.reader(ctx, in, ldr, reader.handles(), forUnmarshal); + } } diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java index cc5f6da0a1b3f..6f2a505f7b94f 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderReader.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.ignite.binary.BinaryObjectException; import org.apache.ignite.internal.binary.BinaryContext; +import org.apache.ignite.internal.binary.BinaryImplUtils; import org.apache.ignite.internal.binary.BinaryObjectEx; import org.apache.ignite.internal.binary.BinaryObjectImpl; import org.apache.ignite.internal.binary.BinaryPositionReadable; @@ -87,7 +88,7 @@ class BinaryBuilderReader implements BinaryPositionReadable { this.arr = other.arr; this.pos = start; - reader = BinaryUtils.reader(ctx, + reader = BinaryImplUtils.reader(ctx, BinaryStreams.inputStream(arr, start), null, other.reader, From c520afa5d9b7df4f57dccceab9aec1f1e39761ae Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 23 Sep 2026 13:19:12 +0300 Subject: [PATCH 18/20] Merge branch 'master' into rmv_th_loc_ign --- .../ignite/internal/binary/BinaryUtils.java | 26 +++---- .../ignite/internal/binary/StringWriter.java | 67 ------------------- 2 files changed, 13 insertions(+), 80 deletions(-) diff --git a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index c4e9f287d2ff3..017fc363e7602 100644 --- a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -2328,10 +2328,10 @@ public static BinaryReaderEx reader(BinaryContext ctx, BinaryInputStream in, Cla * @param forUnmarshal {@code True} if reader is need to unmarshal object. */ static BinaryReaderEx reader(BinaryContext ctx, - BinaryInputStream in, - ClassLoader ldr, - @Nullable BinaryReaderHandles hnds, - boolean forUnmarshal) { + BinaryInputStream in, + ClassLoader ldr, + @Nullable BinaryReaderHandles hnds, + boolean forUnmarshal) { return binariesFactory.reader(ctx, in, ldr, hnds, forUnmarshal); } @@ -2345,10 +2345,10 @@ static BinaryReaderEx reader(BinaryContext ctx, * @param forUnmarshal {@code True} if reader is need to unmarshal object. */ public static BinaryReaderEx reader(BinaryContext ctx, - BinaryInputStream in, - ClassLoader ldr, - boolean skipHdrCheck, - boolean forUnmarshal) { + BinaryInputStream in, + ClassLoader ldr, + boolean skipHdrCheck, + boolean forUnmarshal) { return reader(ctx, in, ldr, null, skipHdrCheck, forUnmarshal); } @@ -2363,11 +2363,11 @@ public static BinaryReaderEx reader(BinaryContext ctx, * @param forUnmarshal {@code True} if reader is need to unmarshal object. */ static BinaryReaderEx reader(BinaryContext ctx, - BinaryInputStream in, - ClassLoader ldr, - @Nullable BinaryReaderHandles hnds, - boolean skipHdrCheck, - boolean forUnmarshal) { + BinaryInputStream in, + ClassLoader ldr, + @Nullable BinaryReaderHandles hnds, + boolean skipHdrCheck, + boolean forUnmarshal) { return binariesFactory.reader(ctx, in, ldr, hnds, skipHdrCheck, forUnmarshal); } diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java index 3e253ce1fd5dd..36e77925cb21b 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java @@ -119,16 +119,11 @@ private static int writeLatin1(byte[] val, BinaryOutputStream out) { if (b >= 0) out.unsafeWriteByte(b); - - utfLen++; - } else { int c = b & 0b1111_1111; out.unsafeWriteByte((byte)(0b1100_0000 | (c >> 6))); out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111))); - - utfLen += 2; } } @@ -149,28 +144,20 @@ private static int writeChars(String val, BinaryOutputStream out) { int start = out.position(); int len = val.length(); - int utfLen = 0; for (int i = 0; i < len; i++) { char c = val.charAt(i); if (c < 0x80) out.unsafeWriteByte((byte)c); - - utfLen++; - } else if (c < 0x800) { out.unsafeWriteByte((byte)(0b11_000000 | (c >> 6))); out.unsafeWriteByte((byte)(0b10_000000 | (c & 0b00_111111))); - - utfLen += 2; } else if (!Character.isSurrogate(c)) { out.unsafeWriteByte((byte)(0b1110_0000 | (c >> 12))); out.unsafeWriteByte((byte)(0b1000_0000 | ((c >> 6) & 0b0011_1111))); out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111))); - - utfLen += 3; } else { char c2; @@ -183,60 +170,6 @@ else if (!Character.isSurrogate(c)) { out.unsafeWriteByte((byte)(0b1000_0000 | ((cp >> 6) & 0b0011_1111))); out.unsafeWriteByte((byte)(0b1000_0000 | (cp & 0b0011_1111))); - utfLen += 4; - i++; - } - else { - out.unsafeWriteByte((byte)'?'); - - utfLen++; - } - } - } - - return utfLen; - } - - /** - * Encodes string chars to the buffer as UTF-8. Replicates {@code String#getBytes(UTF_8)} behavior exactly, - * including replacement of malformed surrogates with {@code '?'}. Buffer capacity must be ensured by the caller. - * - * @param val Value. - * @param out Output stream. - * @return Count of written bytes. - */ - private static int encodeChars(String val, BinaryOutputStream out) { - byte[] buf = out.array(); - int len = val.length(); - - // Unsafe writes skip the array bounds checks: capacity is ensured by the caller. - long off = GridUnsafe.BYTE_ARR_OFF + out.position(); - - for (int i = 0; i < len; i++) { - char c = val.charAt(i); - - if (c < 0x80) - GridUnsafe.putByte(buf, off++, (byte)c); - else if (c < 0x800) { - GridUnsafe.putByte(buf, off++, (byte)(0b1100_0000 | (c >> 6))); - GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (c & 0b0011_1111))); - } - else if (!Character.isSurrogate(c)) { - GridUnsafe.putByte(buf, off++, (byte)(0b1110_0000 | (c >> 12))); - GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | ((c >> 6) & 0b0011_1111))); - GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (c & 0b0011_1111))); - } - else { - char c2; - - if (Character.isHighSurrogate(c) && i + 1 < len && Character.isLowSurrogate(c2 = val.charAt(i + 1))) { - int cp = Character.toCodePoint(c, c2); - - GridUnsafe.putByte(buf, off++, (byte)(0b1111_0000 | (cp >> 18))); - GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | ((cp >> 12) & 0b0011_1111))); - GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | ((cp >> 6) & 0b0011_1111))); - GridUnsafe.putByte(buf, off++, (byte)(0b1000_0000 | (cp & 0b0011_1111))); - i++; } else From a099aee64cef0c2c9635d0cec2289c19ecc22554 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Wed, 23 Sep 2026 15:20:18 +0300 Subject: [PATCH 19/20] Merge branch 'master' into rmv_th_loc_ign --- .../internal/binary/BinaryReaderHandlesHolder.java | 7 ------- .../binary/BinaryReaderHandlesHolderImpl.java | 5 ----- .../ignite/internal/binary/BinaryReaderExImpl.java | 11 +++-------- 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolder.java b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolder.java index 01c05e97dfb4d..5308d9e0766c4 100644 --- a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolder.java +++ b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolder.java @@ -21,13 +21,6 @@ * Holder for handles. */ interface BinaryReaderHandlesHolder { - /** - * Set handle. - * - * @param obj Object. - */ - public void setHandle(Object obj); - /** * Set handle. * diff --git a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolderImpl.java b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolderImpl.java index 0d1c0cd3b91e2..6adb2f2f1a747 100644 --- a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolderImpl.java +++ b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryReaderHandlesHolderImpl.java @@ -24,11 +24,6 @@ class BinaryReaderHandlesHolderImpl implements BinaryReaderHandlesHolder { /** Handles. */ private BinaryReaderHandles hnds; - /** {@inheritDoc} */ - @Override public void setHandle(Object obj) { - throw new UnsupportedOperationException(); - } - /** {@inheritDoc} */ @Override public void setHandle(Object obj, int pos) { handles().put(pos, obj); diff --git a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java index 95679f6041e1d..8b0fe48dbb383 100644 --- a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java +++ b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java @@ -374,11 +374,6 @@ public BinaryReaderExImpl(BinaryContext ctx, return null; } - /** {@inheritDoc} */ - @Override public void setHandle(Object obj) { - setHandle(obj, start); - } - /** {@inheritDoc} */ @Override public void setHandle(Object obj, int pos) { handles().put(pos, obj); @@ -2375,7 +2370,7 @@ private Object read(BinaryClassDescriptor desc) throws BinaryObjectException { case BINARY: res = newInstance(desc.ctor(), desc.describedClass()); - setHandle(res); + setHandle(res, start); if (desc.serializer != null) desc.serializer.readBinary(res, this); @@ -2387,7 +2382,7 @@ private Object read(BinaryClassDescriptor desc) throws BinaryObjectException { case OBJECT: res = newInstance(desc.ctor(), desc.describedClass()); - setHandle(res); + setHandle(res, start); for (BinaryFieldDescriptor info : desc.fields) readField(res, info); @@ -2404,7 +2399,7 @@ private Object read(BinaryClassDescriptor desc) throws BinaryObjectException { try { res = desc.readResolveMtd.invoke(res); - setHandle(res); + setHandle(res, start); } catch (IllegalAccessException e) { throw new RuntimeException(e); From f056c5e4740f3c815b6a19ef1a3cb27c66e8c290 Mon Sep 17 00:00:00 2001 From: Nikolay Izhikov Date: Thu, 24 Sep 2026 12:31:17 +0300 Subject: [PATCH 20/20] IGNITE-27088 Review fixes --- .../ignite/internal/binary/BinaryUtils.java | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java index 017fc363e7602..d3585cb869c35 100644 --- a/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java +++ b/modules/binary/api/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java @@ -129,9 +129,6 @@ TreeMap.class, new BinaryTreeMapWriteReplacer(), TreeSet.class, new BinaryTreeSetWriteReplacer() ); - /** {@code true} if serialized value of this type cannot contain references to objects. */ - private static final boolean[] PLAIN_TYPE_FLAG = new boolean[102]; - /** Binary classes. */ private static final Collection> BINARY_CLS; @@ -236,19 +233,6 @@ TreeSet.class, new BinaryTreeSetWriteReplacer() BINARY_CLS = Set.copyOf(FLAG_TO_CLASS.values()); - for (byte b : new byte[] { - GridBinaryMarshaller.BYTE, GridBinaryMarshaller.SHORT, GridBinaryMarshaller.INT, GridBinaryMarshaller.LONG, - GridBinaryMarshaller.FLOAT, GridBinaryMarshaller.DOUBLE, GridBinaryMarshaller.CHAR, GridBinaryMarshaller.BOOLEAN, - GridBinaryMarshaller.DECIMAL, GridBinaryMarshaller.STRING, GridBinaryMarshaller.UUID, GridBinaryMarshaller.DATE, - GridBinaryMarshaller.TIMESTAMP, GridBinaryMarshaller.TIME, GridBinaryMarshaller.BYTE_ARR, GridBinaryMarshaller.SHORT_ARR, - GridBinaryMarshaller.INT_ARR, GridBinaryMarshaller.LONG_ARR, GridBinaryMarshaller.FLOAT_ARR, GridBinaryMarshaller.DOUBLE_ARR, - GridBinaryMarshaller.TIME_ARR, GridBinaryMarshaller.CHAR_ARR, GridBinaryMarshaller.BOOLEAN_ARR, - GridBinaryMarshaller.DECIMAL_ARR, GridBinaryMarshaller.STRING_ARR, GridBinaryMarshaller.UUID_ARR, GridBinaryMarshaller.DATE_ARR, - GridBinaryMarshaller.TIMESTAMP_ARR, GridBinaryMarshaller.ENUM, GridBinaryMarshaller.ENUM_ARR, GridBinaryMarshaller.NULL}) { - - PLAIN_TYPE_FLAG[b] = true; - } - FIELD_TYPE_NAMES = new String[104]; FIELD_TYPE_NAMES[GridBinaryMarshaller.BYTE] = "byte"; @@ -341,24 +325,6 @@ public static Object unwrapTemporary(Object obj) { return obj; } - /** - * @return {@code true} if content of serialized value cannot contain references to other object. - */ - public static boolean isPlainType(int type) { - return type > 0 && type < PLAIN_TYPE_FLAG.length && PLAIN_TYPE_FLAG[type]; - } - - /** - * Checks whether an array type values can or can not contain references to other object. - * - * @param type Array type. - * @return {@code true} if content of serialized array value cannot contain references to other object. - */ - public static boolean isPlainArrayType(int type) { - return (type >= GridBinaryMarshaller.BYTE_ARR && type <= GridBinaryMarshaller.DATE_ARR) - || type == GridBinaryMarshaller.TIMESTAMP_ARR || type == GridBinaryMarshaller.TIME_ARR; - } - /** * @param cls Class. * @return Binary field type.