Add support for reading Arrow Files (with footer)

[paleolimbot]

We have support for reading the information in the footer:

arrow-nanoarrow/src/nanoarrow/ipc/decoder.c

Lines 1312 to 1345 in bdb7d0c

	ArrowErrorCode ArrowIpcDecoderDecodeFooter(struct ArrowIpcDecoder* decoder,
	struct ArrowBufferView data,
	struct ArrowError* error) {
	struct ArrowIpcDecoderPrivate* private_data =
	(struct ArrowIpcDecoderPrivate*)decoder->private_data;
	int32_t footer_and_size_and_magic_size =
	decoder->header_size_bytes + sizeof(int32_t) + strlen(NANOARROW_IPC_MAGIC);
	const uint8_t* footer_data =
	data.data.as_uint8 + data.size_bytes - footer_and_size_and_magic_size;
	ns(Footer_table_t) footer = ns(Footer_as_root(footer_data));
	NANOARROW_RETURN_NOT_OK(
	ArrowIpcDecoderDecodeSchemaHeader(decoder, ns(Footer_schema(footer)), error));
	NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeSchemaImpl(
	ns(Footer_schema(footer)), &private_data->footer.schema, error));
	ns(Block_vec_t) blocks = ns(Footer_recordBatches(footer));
	int64_t n = ns(Block_vec_len(blocks));
	NANOARROW_RETURN_NOT_OK(ArrowBufferResize(&private_data->footer.record_batch_blocks,
	sizeof(struct ArrowIpcFileBlock) * n,
	/*shrink_to_fit=*/0));
	struct ArrowIpcFileBlock* record_batches =
	(struct ArrowIpcFileBlock*)private_data->footer.record_batch_blocks.data;
	for (int64_t i = 0; i < n; i++) {
	record_batches[i].offset = ns(Block_offset(blocks + i));
	record_batches[i].metadata_length = ns(Block_metaDataLength(blocks + i));
	record_batches[i].body_length = ns(Block_bodyLength(blocks + i));
	}
	decoder->footer = &private_data->footer;
	return NANOARROW_OK;
	}

...but this is not well integrated into the rest of the read process.

duckdb-nanoarrow implements the ability to read an arrow file as if were a stream (by skipping the ARROW1\0\0 at the start of the file and erroring if it sees any dictionaries), which we should possibly implement as part of our stream reading as well (at least as an option).

paleolimbot/duckdb-nanoarrow#14

[paleolimbot]

Also relevant: how to find the footer at the end of the file:

arrow-nanoarrow/src/nanoarrow/integration/ipc_integration.cc

Lines 166 to 194 in bdb7d0c

	// FIXME this API needs to be public; it's a bit smelly to pretend that we support
	// reading files when this bespoke program is the only one which can do it
	//
	// For now: just check the first 8 bytes of the file and read a stream (ignoring the
	// Footer).
	File ipc_file;
	NANOARROW_RETURN_NOT_OK(ipc_file.open(path, "rb", error));
	auto bytes = ipc_file.read();
	auto min_size = sizeof(NANOARROW_IPC_FILE_PADDED_MAGIC) + sizeof(int32_t) +
	strlen(NANOARROW_IPC_FILE_PADDED_MAGIC);
	if (bytes.size() < min_size) {
	ArrowErrorSet(error, "Expected file of more than %lu bytes, got %ld", min_size,
	bytes.size());
	return EINVAL;
	}
	if (memcmp(bytes.data(), NANOARROW_IPC_FILE_PADDED_MAGIC,
	sizeof(NANOARROW_IPC_FILE_PADDED_MAGIC)) != 0) {
	ArrowErrorSet(error, "File did not begin with 'ARROW1\\0\\0'");
	return EINVAL;
	}
	nanoarrow::ipc::UniqueDecoder decoder;
	NANOARROW_RETURN_NOT_OK_WITH_ERROR(ArrowIpcDecoderInit(decoder.get()), error);
	NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderVerifyFooter(
	decoder.get(), {{bytes.data()}, static_cast<int64_t>(bytes.size())}, error));
	NANOARROW_RETURN_NOT_OK(ArrowIpcDecoderDecodeFooter(
	decoder.get(), {{bytes.data()}, static_cast<int64_t>(bytes.size())}, error));