feat: add Parquet reader for Iceberg-backed CloudEvent retrieval

.gitignore

@@ -32,3 +32,4 @@ __debug_bin*
 .env
 
 
+fetch-api

Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean run build install dep test lint format docker
+.PHONY: clean run build install dep test lint format docker gqlgen podman
 
 SHELL := /bin/bash
 PATHINSTBIN = $(abspath ./bin)
@@ -20,7 +20,7 @@ VER_CUT   := $(shell echo $(VERSION) | cut -c2-)
 
 # Dependency versions
 GOLANGCI_VERSION   = latest
-PROTOC_VERSION             = 28.3
+PROTOC_VERSION             = 33.4
 PROTOC_GEN_GO_VERSION      = $(shell go list -m -f '{{.Version}}' google.golang.org/protobuf)
 PROTOC_GEN_GO_GRPC_VERSION = v1.5.1
 
@@ -56,9 +56,26 @@ lint: ## run linter
 	@PATH=$$PATH golangci-lint run --timeout 10m
 
 docker: dep ## build docker image
-	@docker build -f ./Dockerfile . -t dimozone/$(BIN_NAME):$(VER_CUT)
+	@docker build --build-arg APP_NAME=$(BIN_NAME) -f ./Dockerfile . -t dimozone/$(BIN_NAME):$(VER_CUT)
 	@docker tag dimozone/$(BIN_NAME):$(VER_CUT) dimozone/$(BIN_NAME):latest
 
+# Build multi-arch (amd64 + arm64) and push with a random tag. Does not trigger GitHub workflows.
+# Requires: docker buildx, docker login. Run from repo root.
+docker-push-multiarch:
+	$(eval TAG := dev-$(shell openssl rand -hex 6))
+	@echo "Building and pushing dimozone/$(BIN_NAME):$(TAG) (linux/amd64, linux/arm64)"
+	@docker buildx build --build-arg APP_NAME=$(BIN_NAME) --platform linux/amd64,linux/arm64 -f ./Dockerfile --push -t dimozone/$(BIN_NAME):$(TAG) .
+	@echo "Pushed dimozone/$(BIN_NAME):$(TAG)"
+
+# Same as docker-push-multiarch but using podman (manifest list + manifest push --all).
+podman-push-multiarch:
+	$(eval TAG := dev-$(shell openssl rand -hex 6))
+	$(eval IMAGE := dimozone/$(BIN_NAME):$(TAG))
+	@echo "Building and pushing $(IMAGE) (linux/amd64, linux/arm64)"
+	@podman build --build-arg APP_NAME=$(BIN_NAME) --platform linux/amd64,linux/arm64 --manifest $(IMAGE) -f ./Dockerfile .
+	@podman manifest push --all $(IMAGE) docker://$(IMAGE)
+	@echo "Pushed $(IMAGE)"
+
 tools-golangci-lint: ## install golangci-lint
 	@mkdir -p $(PATHINSTBIN)
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | BINARY=golangci-lint bash -s -- ${GOLANGCI_VERSION}
@@ -80,7 +97,10 @@ endif
 
 make tools: tools-golangci-lint tools-protoc ## install all tools
 
-generate: generate-swagger generate-go generate-grpc ## run all file generation for the project
+gqlgen: ## Generate gqlgen code.
+	@go tool gqlgen generate
+
+generate: gqlgen generate-swagger generate-go generate-grpc ## run all file generation for the project
 
 generate-swagger: ## generate swagger documentation
 	@go tool swag -version

charts/fetch-api/values-prod.yaml

@@ -11,7 +11,6 @@ env:
   TOKEN_EXCHANGE_ISSUER_URL: https://auth-roles-rights.dimo.zone
   CLOUDEVENT_BUCKET: dimo-ingest-cloudevent-prod
   EPHEMERAL_BUCKET: dimo-ingest-ephemeral-prod
-  VC_BUCKET: dimo-network-vc-prod
   VEHICLE_NFT_ADDRESS: '0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF'
   CHAIN_ID: 137
 ingress:

charts/fetch-api/values.yaml

@@ -33,7 +33,6 @@ env:
   TOKEN_EXCHANGE_ISSUER_URL: https://auth-roles-rights.dev.dimo.zone
   CLOUDEVENT_BUCKET: dimo-ingest-cloudevent-dev
   EPHEMERAL_BUCKET: dimo-ingest-ephemeral-dev
-  VC_BUCKET: dimo-network-vc-dev
   S3_AWS_REGION: us-east-2
   VEHICLE_NFT_ADDRESS: '0x45fbCD3ef7361d156e8b16F5538AE36DEdf61Da8'
   CHAIN_ID: 80002

docs/docs.go

@@ -397,6 +397,9 @@ const docTemplate = `{
                         "type": "integer"
                     }
                 },
+                "data_base64": {
+                    "type": "string"
+                },
                 "datacontenttype": {
                     "description": "DataContentType is an optional MIME type for the data field. We almost\nalways serialize to JSON and in that case this field is implicitly\n\"application/json\".",
                     "type": "string"
@@ -469,6 +472,9 @@ const docTemplate = `{
                         }
                     ]
                 },
+                "data_base64": {
+                    "type": "string"
+                },
                 "datacontenttype": {
                     "description": "DataContentType is an optional MIME type for the data field. We almost\nalways serialize to JSON and in that case this field is implicitly\n\"application/json\".",
                     "type": "string"

docs/swagger.json

@@ -388,6 +388,9 @@
                         "type": "integer"
                     }
                 },
+                "data_base64": {
+                    "type": "string"
+                },
                 "datacontenttype": {
                     "description": "DataContentType is an optional MIME type for the data field. We almost\nalways serialize to JSON and in that case this field is implicitly\n\"application/json\".",
                     "type": "string"
@@ -460,6 +463,9 @@
                         }
                     ]
                 },
+                "data_base64": {
+                    "type": "string"
+                },
                 "datacontenttype": {
                     "description": "DataContentType is an optional MIME type for the data field. We almost\nalways serialize to JSON and in that case this field is implicitly\n\"application/json\".",
                     "type": "string"

docs/swagger.yaml

@@ -6,6 +6,8 @@ definitions:
         items:
           type: integer
         type: array
+      data_base64:
+        type: string
       datacontenttype:
         description: |-
           DataContentType is an optional MIME type for the data field. We almost
@@ -72,6 +74,8 @@ definitions:
         allOf:
         - $ref: '#/definitions/github_com_DIMO-Network_fetch-api_pkg_eventrepo.ObjectInfo'
         description: Data contains domain-specific information about the event.
+      data_base64:
+        type: string
       datacontenttype:
         description: |-
           DataContentType is an optional MIME type for the data field. We almost

go.mod

@@ -3,30 +3,35 @@ module github.com/DIMO-Network/fetch-api
 go 1.25
 
 require (
+	github.com/99designs/gqlgen v0.17.86
 	github.com/ClickHouse/clickhouse-go/v2 v2.40.1
 	github.com/DIMO-Network/clickhouse-infra v0.0.5
-	github.com/DIMO-Network/cloudevent v0.1.4
+	github.com/DIMO-Network/cloudevent v0.1.6
 	github.com/DIMO-Network/server-garage v0.0.7
 	github.com/DIMO-Network/shared v1.0.7
 	github.com/DIMO-Network/token-exchange-api v0.3.7
+	github.com/apache/arrow-go/v18 v18.5.1
 	github.com/aws/aws-sdk-go-v2 v1.38.3
 	github.com/aws/aws-sdk-go-v2/credentials v1.18.10
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.87.3
 	github.com/ethereum/go-ethereum v1.16.4
 	github.com/gofiber/fiber/v2 v2.52.9
 	github.com/gofiber/swagger v1.1.1
+	github.com/golang-jwt/jwt/v5 v5.3.0
 	github.com/google/go-cmp v0.7.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
 	github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/rs/zerolog v1.34.0
 	github.com/stretchr/testify v1.11.1
 	github.com/swaggo/swag v1.16.6
+	github.com/tidwall/gjson v1.18.0
+	github.com/vektah/gqlparser/v2 v2.5.31
 	github.com/volatiletech/sqlboiler/v4 v4.19.1
 	go.uber.org/mock v0.6.0
-	golang.org/x/sync v0.17.0
-	google.golang.org/grpc v1.75.0
-	google.golang.org/protobuf v1.36.10
+	golang.org/x/sync v0.19.0
+	google.golang.org/grpc v1.78.0
+	google.golang.org/protobuf v1.36.11
 )
 
 require (
@@ -36,7 +41,9 @@ require (
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/MicahParks/keyfunc/v2 v2.1.0 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/agnivade/levenshtein v1.2.1 // indirect
 	github.com/andybalholm/brotli v1.2.0 // indirect
+	github.com/apache/thrift v0.22.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.6 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.6 // indirect
@@ -74,14 +81,22 @@ require (
 	github.com/go-openapi/jsonreference v0.20.4 // indirect
 	github.com/go-openapi/spec v0.20.14 // indirect
 	github.com/go-openapi/swag v0.22.9 // indirect
+	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/goccy/go-json v0.10.5 // indirect
+	github.com/goccy/go-yaml v1.19.2 // indirect
 	github.com/gofiber/contrib/jwt v1.1.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang-jwt/jwt/v5 v5.3.0 // indirect
+	github.com/golang/snappy v1.0.0 // indirect
+	github.com/google/flatbuffers v25.12.19+incompatible // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/gorilla/websocket v1.5.1 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/holiman/uint256 v1.3.2 // indirect
 	github.com/joho/godotenv v1.5.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/asmfmt v1.3.2 // indirect
+	github.com/klauspost/compress v1.18.2 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect
 	github.com/magiconair/properties v1.8.10 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
@@ -90,6 +105,8 @@ require (
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/mdelapenya/tlscert v0.2.0 // indirect
 	github.com/mfridman/interpolate v0.0.2 // indirect
+	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
+	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/go-archive v0.1.0 // indirect
 	github.com/moby/patternmatcher v0.6.0 // indirect
@@ -102,9 +119,9 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/paulmach/orb v0.11.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/pierrec/lz4/v4 v4.1.23 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
 	github.com/pressly/goose/v3 v3.24.3 // indirect
 	github.com/prometheus/client_golang v1.23.0 // indirect
@@ -118,45 +135,50 @@ require (
 	github.com/shirou/gopsutil/v4 v4.25.5 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/sosodev/duration v1.3.1 // indirect
 	github.com/spf13/cast v1.7.0 // indirect
 	github.com/swaggo/files/v2 v2.0.2 // indirect
 	github.com/testcontainers/testcontainers-go v0.38.0 // indirect
 	github.com/testcontainers/testcontainers-go/modules/clickhouse v0.38.0 // indirect
-	github.com/tidwall/gjson v1.18.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
 	github.com/tklauser/go-sysconf v0.3.15 // indirect
 	github.com/tklauser/numcpus v0.10.0 // indirect
 	github.com/urfave/cli/v2 v2.27.5 // indirect
+	github.com/urfave/cli/v3 v3.6.1 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.65.0 // indirect
 	github.com/volatiletech/inflect v0.0.1 // indirect
 	github.com/volatiletech/strmangle v0.0.7-0.20240503230658-86517898275a // indirect
 	github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
-	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect
-	go.opentelemetry.io/otel v1.37.0 // indirect
-	go.opentelemetry.io/otel/metric v1.37.0 // indirect
-	go.opentelemetry.io/otel/trace v1.37.0 // indirect
+	go.opentelemetry.io/otel v1.38.0 // indirect
+	go.opentelemetry.io/otel/metric v1.38.0 // indirect
+	go.opentelemetry.io/otel/trace v1.38.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.43.0 // indirect
-	golang.org/x/mod v0.28.0 // indirect
-	golang.org/x/net v0.45.0 // indirect
-	golang.org/x/sys v0.37.0 // indirect
-	golang.org/x/text v0.30.0 // indirect
-	golang.org/x/tools v0.37.0 // indirect
-	golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20250818200422-3122310a409c // indirect
+	golang.org/x/crypto v0.47.0 // indirect
+	golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect
+	golang.org/x/mod v0.32.0 // indirect
+	golang.org/x/net v0.49.0 // indirect
+	golang.org/x/sys v0.40.0 // indirect
+	golang.org/x/telemetry v0.0.0-20260109210033-bd525da824e2 // indirect
+	golang.org/x/text v0.33.0 // indirect
+	golang.org/x/tools v0.41.0 // indirect
+	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
 tool (
+	github.com/99designs/gqlgen
 	github.com/swaggo/swag/cmd/swag
 	go.uber.org/mock/mockgen
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc