Skip to content
This repository was archived by the owner on Jan 28, 2026. It is now read-only.

demo: adds encryptd for timelock encryption at upload #17

Open
andrewxhill wants to merge 1 commit into
base: main
Choose a base branch
from
Open

demo: adds encryptd for timelock encryption at upload #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 22 additions & 6 deletions cmd/basin/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,23 @@ import (

// DefaultProviderHost is the address of Basin Provider.
const DefaultProviderHost = "basin.tableland.xyz:3000"
const DefaultTlockHost = "https://api.drand.sh/"
const DefaultTlockChain = "dbd506d6ef76e5f386f41c651dcb808c5bcbd75471cc4eafa3f4df7ad4e4c493"

type config struct {
Publications map[string]publication `yaml:"publications"`
}

type publication struct {
User string `yaml:"user"`
Password string `yaml:"password"`
Host string `yaml:"host"`
Port int `yaml:"port"`
Database string `yaml:"database"`
ProviderHost string `yaml:"provider_host"`
User string `yaml:"user"`
Password string `yaml:"password"`
Host string `yaml:"host"`
Port int `yaml:"port"`
Database string `yaml:"database"`
ProviderHost string `yaml:"provider_host"`
TlockDuration string `yaml:"tlock_duration"`
TlockHost string `yaml:"tlock_host"`
TlockChain string `yaml:"tlock_chain"`
}

func newConfig() *config {
Expand All @@ -42,6 +47,17 @@ func loadConfig(path string) (*config, error) {
return &config{}, err
}

// Adding defaults for old configs
for key, pub := range conf.Publications {
if pub.TlockHost == "" {
pub.TlockHost = DefaultTlockHost
}
if pub.TlockChain == "" {
pub.TlockChain = DefaultTlockChain
}
conf.Publications[key] = pub
}
Copy link
Copy Markdown
Contributor Author

@andrewxhill andrewxhill Nov 10, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just reverse compat with existing configs


return conf, nil
}

Expand Down
120 changes: 110 additions & 10 deletions cmd/basin/publication.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,16 @@ import (
"context"
"errors"
"fmt"
"io"
"os"
"path"
"regexp"
"strings"
"time"

"capnproto.org/go/capnp/v3"
"github.com/drand/tlock"
"github.com/drand/tlock/networks/http"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/jackc/pgx/v5"
Expand Down Expand Up @@ -47,7 +51,7 @@ func newPublicationCommand() *cli.Command {
}

func newPublicationCreateCommand() *cli.Command {
var owner, dburi, provider string
var owner, dburi, provider, tlockDuration, tlockHost, tlockChain string
var secure bool

return &cli.Command{
Expand Down Expand Up @@ -77,6 +81,24 @@ func newPublicationCreateCommand() *cli.Command {
Destination: &secure,
Value: true,
},
&cli.StringFlag{
Name: "tlockDuration",
Usage: "The timelock encryption duration (default is no encryption)",
Destination: &tlockDuration,
Required: false,
},
&cli.StringFlag{
Name: "tlockHost",
Usage: "The drand host for timelock encryption",
Destination: &tlockHost,
Value: DefaultTlockHost,
},
&cli.StringFlag{
Name: "tlockChain",
Usage: "The drand chain for timelock encryption",
Destination: &tlockChain,
Value: DefaultTlockChain,
},
},
Action: func(cCtx *cli.Context) error {
if cCtx.NArg() != 1 {
Expand Down Expand Up @@ -117,12 +139,15 @@ func newPublicationCreateCommand() *cli.Command {
}

cfg.Publications[pub] = publication{
Host: pgConfig.Host,
Port: int(pgConfig.Port),
User: pgConfig.User,
Password: pgConfig.Password,
Database: pgConfig.Database,
ProviderHost: provider,
Host: pgConfig.Host,
Port: int(pgConfig.Port),
User: pgConfig.User,
Password: pgConfig.Password,
Database: pgConfig.Database,
ProviderHost: provider,
TlockDuration: tlockDuration,
TlockHost: tlockHost,
TlockChain: tlockChain,
}

if err := yaml.NewEncoder(f).Encode(cfg); err != nil {
Expand Down Expand Up @@ -222,12 +247,12 @@ func newPublicationStartCommand() *cli.Command {
}

func newPublicationUploadCommand() *cli.Command {
var privateKey, publicationName string
var privateKey, publicationName, tlockDuration, encryptPath string
var secure bool

return &cli.Command{
Name: "upload",
Usage: "upload a Parquet file",
Usage: "upload a Parquet file with optional tlock encryption",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "private-key",
Expand All @@ -247,6 +272,18 @@ func newPublicationUploadCommand() *cli.Command {
Destination: &secure,
Value: true,
},
&cli.StringFlag{
Name: "encryptd",
Usage: "optional timelock encryption duration (e.g., '24hr'). If set, the file will be encrypted with tlock before upload",
Destination: &tlockDuration,
Required: false,
},
&cli.StringFlag{
Name: "encryptp",
Usage: "file path to maintain a local copy of the encrypted file",
Destination: &encryptPath,
Required: false,
},
},
Action: func(cCtx *cli.Context) error {
if cCtx.NArg() != 1 {
Expand All @@ -272,14 +309,78 @@ func newPublicationUploadCommand() *cli.Command {
return fmt.Errorf("load config: %s", err)
}

// Use global tlock duration if not supplied
if tlockDuration == "" && cfg.Publications[publicationName].TlockDuration != "" {
tlockDuration = cfg.Publications[publicationName].TlockDuration
}

if encryptPath != "" && tlockDuration == "" {
return fmt.Errorf("error: requesting stored copy without requesting encryption")
}

bp, err := basinprovider.New(cCtx.Context, cfg.Publications[publicationName].ProviderHost, secure)
if err != nil {
return err
}
defer bp.Close()

// Setup the encrpytion dependant on tlockDuration
var encryptor tlock.Tlock
var roundNumber uint64
if tlockDuration != "" {

tl, err := time.ParseDuration(tlockDuration)
if err != nil {
return fmt.Errorf("encryption duration error: %s", err)
}
// Construct a network that can talk to a drand network. Example using the mainnet fastnet network.
network, err := http.NewNetwork(cfg.Publications[publicationName].TlockHost, cfg.Publications[publicationName].TlockChain)
if err != nil {
return fmt.Errorf("drand network error: %s", err)
}

// Use the network to identify the round number that represents the duration.
roundNumber = network.RoundNumber(time.Now().Add(tl))
// Initialize the tlock encryptor with the provided duration
encryptor = tlock.New(network)
}

filepath := cCtx.Args().First()

// Check if we have encrypted content to upload
if tlockDuration != "" {
f, err := os.Open(filepath)
if err != nil {
return fmt.Errorf("open file: %s", err)
}

var encrypted *os.File
if encryptPath == "" {
// Write the encrypted content to a temporary file
encrypted, err = os.CreateTemp("", "encrypted-*")
if err != nil {
return fmt.Errorf("create temp file: %s", err)
}
defer encrypted.Close()
defer os.Remove(encrypted.Name()) // Clean up the file afterwards
} else {
// Write the encrypted content to a temporary file
encrypted, err = os.Create(encryptPath)
if err != nil {
return fmt.Errorf("create output file: %s", err)
}
defer encrypted.Close()
}

// Encrypt the file content using tlock
if err := encryptor.Encrypt(encrypted, io.ReadCloser(f), roundNumber); err != nil {
return fmt.Errorf("encrypt file: %s", err)
}

// Update the filepath to point to the temporary encrypted file
filepath = encrypted.Name()
}

f, err := os.Open(filepath)
if err != nil {
return fmt.Errorf("open file: %s", err)
Expand All @@ -302,7 +403,6 @@ func newPublicationUploadCommand() *cli.Command {
if err := basinStreamer.Upload(cCtx.Context, filepath, bar); err != nil {
return fmt.Errorf("upload: %s", err)
}

return nil
},
}
Expand Down
39 changes: 32 additions & 7 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,62 +9,87 @@ require (
github.com/google/go-cmp v0.5.9
github.com/jackc/pglogrepl v0.0.0-20230728225306-38e8a4e50913
github.com/jackc/pgx/v5 v5.4.3
github.com/lib/pq v1.2.0
github.com/lib/pq v1.10.9
github.com/mitchellh/go-homedir v1.1.0
github.com/olekukonko/tablewriter v0.0.5
github.com/ory/dockertest/v3 v3.10.0
github.com/schollz/progressbar/v3 v3.13.1
github.com/stretchr/testify v1.8.4
github.com/urfave/cli/v2 v2.25.7
golang.org/x/crypto v0.9.0
golang.org/x/crypto v0.13.0
Copy link
Copy Markdown
Contributor Author

@andrewxhill andrewxhill Nov 10, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can rollback these updates. hit errors w/o so would need to investigate

golang.org/x/exp v0.0.0-20230905200255-921286631fa9
google.golang.org/grpc v1.57.0
gopkg.in/yaml.v3 v3.0.1
)

require (
filippo.io/age v1.1.1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/containerd/continuity v0.3.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/creack/pty v1.1.17 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
github.com/docker/cli v20.10.17+incompatible // indirect
github.com/docker/docker v20.10.7+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/drand/drand v1.5.4 // indirect
github.com/drand/kyber v1.2.0 // indirect
github.com/drand/kyber-bls12381 v0.2.6 // indirect
github.com/drand/tlock v1.0.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/holiman/uint256 v1.2.3 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/kilic/bls12-381 v0.1.0 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect
github.com/nikkolasg/hexjson v0.1.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.2 // indirect
github.com/opencontainers/runc v1.1.5 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.15.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.12.0 // indirect
golang.org/x/term v0.8.0 // indirect
golang.org/x/sys v0.14.0 // indirect
golang.org/x/term v0.14.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/tools v0.13.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
zenhack.net/go/util v0.0.0-20230414204917-531d38494cf5 // indirect
)
Loading