Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 20 additions & 47 deletions packages/kafka/src/deserializer/protobuf.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,28 +27,24 @@ const createDeserializer = async (): Promise<ProtobufDeserializer> => {
);

/**
* Default order of varint types used in Protobuf to attempt deserializing Confluent Schema Registry messages.
*/
const varintOrder: Array<'int32' | 'sint32'> = ['int32', 'sint32'];

/**
* Clip the Confluent Schema Registry buffer to remove the index bytes.
* Skip the Confluent Schema Registry message indexes at the start of the buffer.
*
* Confluent writes the indexes as an array of zigzag varints: the array length
* first, then each index. The common case `[0]` is written as a single `0` byte.
*
* @param buffer - The buffer to clip.
* @param intType - The type of the integer to read from the buffer, either 'int32' or 'sint32'.
* @param buffer - The buffer to read from, positioned after the schema ID.
*/
const clipConfluentSchemaRegistryBuffer = (
buffer: Buffer,
intType: 'int32' | 'sint32'
) => {
const skipConfluentMessageIndexes = (buffer: Buffer) => {
const reader = new BufferReader(buffer);
/**
* Read the first varint byte to get the index count or 0.
* Doing so, also advances the reader position to the next byte after the index count.
*/
const indexCount = intType === 'int32' ? reader.int32() : reader.sint32();
// Skip the index bytes
reader.skip(indexCount);
const indexCount = reader.sint32();
Comment thread
dreamorosi marked this conversation as resolved.
Resolved
if (indexCount < 0) {
throw new Error(
`Invalid Confluent message index array length: ${indexCount}`
);
}
for (let i = 0; i < indexCount; i++) {
reader.sint32();
}
return reader;
};

Expand Down Expand Up @@ -83,39 +79,16 @@ const createDeserializer = async (): Promise<ProtobufDeserializer> => {
reader.uint32();
return messageType.decode(reader);
}
/**
* If schemaId is numeric, inferred from its length, we know it's coming from Confluent Schema Registry,
* so we need to remove the message index bytes before decoding.
*/
return messageType.decode(skipConfluentMessageIndexes(buffer));
} catch (error) {
throw new KafkaConsumerDeserializationError(
Comment thread
dreamorosi marked this conversation as resolved.
Resolved
`Failed to deserialize Protobuf message: ${error}, message: ${data}, messageType: ${JSON.stringify(messageType)}`
);
}

/**
* If schemaId is numeric, inferred from its length, we know it's coming from Confluent Schema Registry,
* so we need to remove the MessageIndex bytes.
* We don't know the type of the index, so we try both `int32` and `sint32`. If both fail, we throw an error.
*/
try {
const newBuffer = clipConfluentSchemaRegistryBuffer(
buffer,
varintOrder[0]
);
return messageType.decode(newBuffer);
} catch (error) {
try {
const newBuffer = clipConfluentSchemaRegistryBuffer(
buffer,
varintOrder[1]
);
const decoded = messageType.decode(newBuffer);
// swap varint order if the first attempt failed so we can use the correct one for subsequent messages
varintOrder.reverse();
return decoded;
} catch {
throw new KafkaConsumerDeserializationError(
`Failed to deserialize Protobuf message: ${error}, message: ${data}, messageType: ${JSON.stringify(messageType)}`
);
}
}
};

return deserializer;
Expand Down
28 changes: 22 additions & 6 deletions packages/kafka/tests/unit/consumer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -495,7 +495,7 @@ describe('Kafka consumer', () => {
event.records['mytopic-0'][1].value = 'AAjpBxIGTGFwdG9wGVK4HoXrP49A';
event.records['mytopic-0'][1].valueSchemaMetadata.schemaId =
'1111111111111111';
event.records['mytopic-0'][2].value = 'AgEACOkHEgZMYXB0b3AZUrgehes/j0A=';
event.records['mytopic-0'][2].value = 'BAIACOkHEgZMYXB0b3AZUrgehes/j0A=';
event.records['mytopic-0'][2].valueSchemaMetadata.schemaId = '1';

const handler = kafkaConsumer(
Expand Down Expand Up @@ -552,6 +552,9 @@ describe('Kafka consumer', () => {
type: SchemaType.PROTOBUF,
schema: com.example.protobuf.UserProfile,
parserSchema: z.object({
userId: z.string(),
name: z.string(),
email: z.string(),
address: z.object({
city: z.string(),
street: z.string(),
Expand All @@ -575,11 +578,24 @@ describe('Kafka consumer', () => {
| Error[];

// Assess
expect(result).toHaveLength(4);
expect(result[0]).not.toBeInstanceOf(Error);
expect(result[1]).not.toBeInstanceOf(Error);
expect(result[2]).not.toBeInstanceOf(Error);
expect(result[3]).not.toBeInstanceOf(Error);
const profile = {
name: 'Bob',
email: 'bob@example.com',
age: 28,
isActive: true,
signupDate: '2024-02-02',
tags: ['tag1', 'tag2'],
preferences: { theme: 'light' },
address: { street: '123 Ave', city: 'Seattle', zip: '98101' },
};
expect(result).toEqual(
[
'a8e40971-1552-420d-a7c9-b8982325702d',
'4dcfc61b-3993-49c3-a04f-8a6c7aaf7881',
'2a861628-0800-4b76-bd3f-6ecba7cd286c',
'71320c73-ec57-446e-bd5b-9252d6943182',
].map((userId) => ({ userId, ...profile }))
);
});

it('handles complex protobuf messages from Glue Schema Registry', async () => {
Expand Down
38 changes: 38 additions & 0 deletions packages/kafka/tests/unit/deserializer.protobuf.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,44 @@ describe('Protobuf deserialiser: ', () => {
expect(result).toEqual(expected);
});

it.each([
{ indexes: '[0]', data: 'AAjpBxIGTGFwdG9wGVK4HoXrP49A' },
{ indexes: '[1]', data: 'AgII6QcSBkxhcHRvcBlSuB6F6z+PQA==' },
{ indexes: '[2]', data: 'AgQI6QcSBkxhcHRvcBlSuB6F6z+PQA==' },
{ indexes: '[1, 0]', data: 'BAIACOkHEgZMYXB0b3AZUrgehes/j0A=' },
{ indexes: '[70]', data: 'AowBCOkHEgZMYXB0b3AZUrgehes/j0A=' },
])(
'skips the Confluent message indexes $indexes before decoding',
({ data }) => {
// Prepare
const expected = { id: 1001, name: 'Laptop', price: 999.99 };
const schemaMetadata = {
dataFormat: 'PROTOBUF',
schemaId: '1',
};

// Act
const result = deserialize(data, Product, schemaMetadata);

// Assess
expect(result).toEqual(expected);
}
);

it('throws if the Confluent message index count is negative', () => {
// Prepare
const data = 'AQjpBxIGTGFwdG9wGVK4HoXrP49A'; // 0x01 (zigzag -1) + the Product payload
const schemaMetadata = {
dataFormat: 'PROTOBUF',
schemaId: '1',
};

// Act & Assess
expect(() => deserialize(data, Product, schemaMetadata)).toThrow(
'Invalid Confluent message index array length: -1'
);
});

it('throws if unable to parse a Confluent Schema Registry protobuf', () => {
// Prepare
const data = 'COkHEgZMYXB0b3AZUrgehes/j0A=';
Expand Down
Loading