Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,7 @@
* An implementation of {@link BigQueryBaseArray} used to represent Array values from Arrow data.
*/
class BigQueryArrowArray extends BigQueryBaseArray {
private static final BigQueryTypeCoercer BIGQUERY_TYPE_COERCER =
BigQueryTypeCoercionUtility.INSTANCE;

private JsonStringArrayList<?> values;

public BigQueryArrowArray(Field schema, JsonStringArrayList<?> values) {
Expand All @@ -43,7 +42,7 @@ public BigQueryArrowArray(
}

@Override
public Object getArray() {
public Object getArray() throws SQLException {
LOG.finestTrace("getArray");
ensureValid();
if (values == null) {
Expand All @@ -53,7 +52,7 @@ public Object getArray() {
}

@Override
public Object getArray(long index, int count) {
public Object getArray(long index, int count) throws SQLException {
LOG.finestTrace("getArray");
ensureValid();
if (values == null) {
Expand Down Expand Up @@ -98,12 +97,12 @@ public void free() {
}

@Override
Object getCoercedValue(int index) {
Object getCoercedValue(int index) throws SQLException {
LOG.finestTrace("getCoercedValue");
Object value = this.values.get(index);
return this.arrayOfStruct
? new BigQueryArrowStruct(
schema.getSubFields(), (JsonStringHashMap<?, ?>) value, this.LOG.getArrowStructLogger())
: BIGQUERY_TYPE_COERCER.coerceTo(getTargetClass(), value, this.LOG);
: BigQueryTypeRegistry.convert(value, getTargetClass());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,17 +31,18 @@
import io.opentelemetry.context.Scope;
import java.io.IOException;
import java.math.BigDecimal;
import java.sql.Date;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Future;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.DateDayVector;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.VectorLoader;
import org.apache.arrow.vector.VectorSchemaRoot;
Expand Down Expand Up @@ -340,6 +341,9 @@ private Object getObjectInternal(int columnIndex) throws SQLException {
FieldVector currentColumn = this.vectorSchemaRoot.getVector(columnIndex - 1);
// get the current row
value = currentColumn.getObject(this.currentBatchRowIndex);
if (value instanceof Integer && currentColumn instanceof DateDayVector) {
value = LocalDate.ofEpochDay(((Integer) value).longValue());
}
}
setWasNull(value);
return value;
Expand All @@ -357,7 +361,7 @@ public Object getObject(int columnIndex) throws SQLException {
}

if (this.isNested && columnIndex == 1) {
return this.bigQueryTypeCoercer.coerceTo(Integer.class, value, this.LOG);
return BigQueryTypeRegistry.convert(value, Integer.class);
}

if (this.isNested && columnIndex == 2) {
Expand All @@ -368,10 +372,11 @@ public Object getObject(int columnIndex) throws SQLException {
(JsonStringHashMap<?, ?>) value,
this.LOG.getArrowStructLogger());
}
Class<?> targetClass =
BigQueryJdbcTypeMappings.standardSQLToJavaTypeMapping.get(
arrayField.getType().getStandardType());
return this.bigQueryTypeCoercer.coerceTo(targetClass, value, this.LOG);
if (value instanceof Integer
&& arrayField.getType().getStandardType() == StandardSQLTypeName.DATE) {
value = LocalDate.ofEpochDay(((Integer) value).longValue());
}
return BigQueryTypeRegistry.convert(value, arrayField.getType().getStandardType(), null);
}

int fieldIndex = this.isNested ? 0 : columnIndex - 1;
Expand Down Expand Up @@ -437,10 +442,7 @@ public Object getObject(int columnIndex) throws SQLException {
// Strip trailing zeros to match JSON API and CLI output
return ((BigDecimal) value).stripTrailingZeros();
}
Class<?> targetClass =
BigQueryJdbcTypeMappings.standardSQLToJavaTypeMapping.get(
fieldSchema.getType().getStandardType());
return this.bigQueryTypeCoercer.coerceTo(targetClass, value, this.LOG);
return BigQueryTypeRegistry.convert(value, fieldSchema.getType().getStandardType(), null);
}
}

Expand All @@ -460,24 +462,23 @@ private StandardSQLTypeName getElementTypeFromValue(Object element) {
return StandardSQLTypeName.STRING;
}

private String formatRangeElement(Object element, StandardSQLTypeName elementType) {
private String formatRangeElement(Object element, StandardSQLTypeName elementType)
throws SQLException {
if (element == null) {
return "UNBOUNDED";
}
switch (elementType) {
case DATE:
// Arrow gives DATE as an Integer (days since epoch)
Date date = this.bigQueryTypeCoercer.coerceTo(Date.class, (Integer) element, this.LOG);
return date.toString();
return LocalDate.ofEpochDay(((Integer) element).longValue()).toString();
case DATETIME:
// Arrow gives DATETIME as a LocalDateTime
Timestamp dtTs =
this.bigQueryTypeCoercer.coerceTo(Timestamp.class, (LocalDateTime) element, this.LOG);
return this.bigQueryTypeCoercer.coerceTo(String.class, dtTs, this.LOG);
Timestamp dtTs = BigQueryTypeRegistry.convert((LocalDateTime) element, Timestamp.class);
return BigQueryTypeRegistry.convert(dtTs, String.class);
case TIMESTAMP:
// Arrow gives TIMESTAMP as a Long (microseconds since epoch)
Timestamp ts = this.bigQueryTypeCoercer.coerceTo(Timestamp.class, (Long) element, this.LOG);
return this.bigQueryTypeCoercer.coerceTo(String.class, ts, this.LOG);
Timestamp ts = BigQueryTypeRegistry.convert((Long) element, Timestamp.class);
return BigQueryTypeRegistry.convert(ts, String.class);
default:
// Fallback for any other unexpected type
return element.toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@

import com.google.cloud.bigquery.Field;
import com.google.cloud.bigquery.FieldList;
import com.google.cloud.bigquery.StandardSQLTypeName;
import java.lang.reflect.Array;
import java.sql.SQLException;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import org.apache.arrow.vector.util.JsonStringArrayList;
Expand All @@ -30,8 +33,6 @@
* An implementation of {@link BigQueryBaseStruct} used to represent Struct values from Arrow data.
*/
class BigQueryArrowStruct extends BigQueryBaseStruct {
private static final BigQueryTypeCoercer BIGQUERY_TYPE_COERCER =
BigQueryTypeCoercionUtility.INSTANCE;

private final FieldList schema;

Expand All @@ -54,7 +55,7 @@ FieldList getSchema() {
}

@Override
public Object[] getAttributes() {
public Object[] getAttributes() throws SQLException {
LOG.finestTrace("getAttributes");
int size = this.schema.size();
Object[] attributes = (Object[]) Array.newInstance(Object.class, size);
Expand All @@ -73,7 +74,7 @@ public Object[] getAttributes() {
return attributes;
}

private Object getValue(Field currentSchema, Object currentValue) {
private Object getValue(Field currentSchema, Object currentValue) throws SQLException {
LOG.finestTrace("getValue");
if (isArray(currentSchema)) {
return new BigQueryArrowArray(
Expand All @@ -84,10 +85,12 @@ private Object getValue(Field currentSchema, Object currentValue) {
(JsonStringHashMap<?, ?>) currentValue,
this.LOG.getArrowStructLogger());
} else {
Class<?> targetClass =
BigQueryJdbcTypeMappings.standardSQLToJavaTypeMapping.get(
currentSchema.getType().getStandardType());
return BIGQUERY_TYPE_COERCER.coerceTo(targetClass, currentValue, this.LOG);
if (currentValue instanceof Integer
&& currentSchema.getType().getStandardType() == StandardSQLTypeName.DATE) {
currentValue = LocalDate.ofEpochDay(((Integer) currentValue).longValue());
}
return BigQueryTypeRegistry.convert(
currentValue, currentSchema.getType().getStandardType(), null);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,7 @@ public final String getBaseTypeName() {
public final int getBaseType() {
LOG.finestTrace("getBaseType");
ensureValid();
return BigQueryJdbcTypeMappings.standardSQLToJavaSqlTypesMapping.get(
schema.getType().getStandardType());
return BigQueryTypeRegistry.toJdbcType(schema.getType().getStandardType());
}

@Override
Expand All @@ -91,7 +90,7 @@ public final ResultSet getResultSet(long index, int count, Map<String, Class<?>>
throw new BigQueryJdbcSqlFeatureNotSupportedException(CUSTOMER_TYPE_MAPPING_NOT_SUPPORTED);
}

protected Object getArrayInternal(int fromIndex, int toIndexExclusive) {
protected Object getArrayInternal(int fromIndex, int toIndexExclusive) throws SQLException {
LOG.finestTrace("getArrayInternal");
Class<?> targetClass = getTargetClass();
int size = toIndexExclusive - fromIndex;
Expand Down Expand Up @@ -145,11 +144,10 @@ protected Class<?> getTargetClass() {
LOG.finestTrace("getTargetClass");
return this.arrayOfStruct
? Struct.class
: BigQueryJdbcTypeMappings.standardSQLToJavaTypeMapping.get(
this.schema.getType().getStandardType());
: BigQueryTypeRegistry.toJavaClass(this.schema.getType().getStandardType());
}

abstract Object getCoercedValue(int index);
abstract Object getCoercedValue(int index) throws SQLException;

static boolean isArray(Field currentSchema) {
return currentSchema.getMode() == REPEATED;
Expand Down
Loading
Loading