bootstrap-aptos-fullnode/haproxy-fullnode.cfg at master · bigbangbtc/bootstrap-aptos-fullnode · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
global
    log stdout len 10240 format raw local0

    # Config manual: https://cbonte.github.io/haproxy-dconv/2.5/configuration.html
    # magic values : terraform/helm/aptos-node/values.yaml

    maxconn 8192
    # This limits the whole HA Proxy impacting both fullnodes and other frontends
    # maxconnrate 128
    nbthread 4

    #4MB for client facing sndbuf/rcvbuf. -- 100Mb/s with 300 mili latency (e.g., us-asia)
    tune.sndbuf.client 4194304	#tcpBufSize
    tune.rcvbuf.client 4194304	#tcpBufSize

    user nobody

## TCP port defaults
defaults
    log global
    mode tcp
    #option tcplog
    option dontlog-normal
    log-format "%ci:%cp - %sp[%rt] [%t] %ft %Tw/%Tc/%Tt %B [%ts] %ac/%fc/%bc/%sc/%rc %sq/%bq"
    maxconn 8192		#Validator network mesh + FN x2
    retries 3
    timeout queue 5s  #limits num of concurrent connections. Not clear if t/o connect is needed. #https://www.papertrail.com/solution/tips/haproxy-logging-how-to-tune-timeouts-for-performance/
    timeout connect 5s
    # enough for 1 successfull + 5 unsuccessfull HB(10 sec interval) + 20 sec timeout
    timeout server 80s
    timeout client 80s

    timeout client-fin 3s #How long to hold an interrupted client connection.
    timeout server-fin 1s

frontend fullnode-fn
    mode tcp
    bind :6182
    default_backend fullnode

    # Deny requests from blocked IPs
    tcp-request connection reject if { src -n -f /usr/local/etc/haproxy/blocked.ips }

    # We deem a connection rate high when an IP is attempting to reconnect more than twice a min
    acl ip_high_conn_rate sc0_conn_rate gt 12

    stick-table type ip size 128K expire 30m store gpc1,conn_rate(1m),bytes_out_cnt	##about 500MB of memory
    tcp-request connection track-sc0 src 						   #update table with src ip as key, store in sc0

    #We Count rate-limit manualy -- Will be more CPU intensieve but will allow whitelists to enter and up to rateLimitSession non blacklisted IPs.
    tcp-request connection track-sc1 int(1) table CONN_RATE

    #This connection is silently dropped no reason to count it for rateLimitSession
    tcp-request connection sc-inc-gpc1(1) unless ip_high_conn_rate

    # IP rejeceted due to to many unsucsessfull tcp attempts
    #-1- Enforce connection rate limit
    tcp-request connection silent-drop if ip_high_conn_rate

    #an IP that had a sucessfull connection.
    #-2- Allow Whitelist
    tcp-request connection accept if { sc0_get_gpc1() ge 1 }

    #-3- Enforce RateLimit. Connection attempts by *new* IPs/sec
    tcp-request connection reject if { sc1_gpc1_rate(CONN_RATE) gt  256 } #rateLimitSession

    # This is a successfull connection i.e., was sent more than 16K bytes in the last 30 min
    #tcp-request session sc-set-gpt0(0) int(...)  if { sc0_kbytes_out gt 16 }
    #<2> Mark Whitelist
    tcp-request session sc-inc-gpc1(0) if { sc0_kbytes_out gt 4 }


backend fullnode
    default-server maxconn 128
    server fullnode fullnode:6182

#CONNRATE holds only entry with key 1: used for determening global conn rate
backend CONN_RATE
    stick-table type integer size 1 expire 1m store gpc1,gpc1_rate(1s)

##################  HTTP: metrics & API
defaults
        retries 3
        timeout queue 5s  #limits num of concurrent connections. Not clear if t/o connect is needed. #https://www.papertrail.com/solution/tips/haproxy-logging-how-to-tune-timeouts-for-performance/
        timeout connect 5s
        timeout server 60s #what makes sense? for silence between nodes?
        timeout client 60s

        timeout client-fin 3s #How long to hold an interrupted client connection.
        timeout server-fin 1s

	timeout http-request 60s #len of http request
	timeout http-keep-alive 2s

        rate-limit sessions 256

frontend fullnode-metrics
    mode http
    option httplog
    bind :9101
    default_backend fullnode-metrics

    # Deny requests from blocked IPs
    tcp-request connection reject if { src -n -f /usr/local/etc/haproxy/blocked.ips }
    http-request add-header Forwarded "for=%ci"

backend fullnode-metrics
    mode http
    default-server maxconn 16
    server fullnode fullnode:9101

frontend fullnode-api
    mode http
    option httplog
    bind :8080
    default_backend fullnode-api

    # Deny requests from blocked IPs
    tcp-request connection reject if { src -n -f /usr/local/etc/haproxy/blocked.ips }
    http-request add-header Forwarded "for=%ci"

backend fullnode-api
    mode http
    default-server maxconn 16
    server fullnode fullnode:8080

frontend stats
    mode http
    bind :9102
    option http-use-htx
    http-request use-service prometheus-exporter if { path /metrics }
    stats enable
    stats uri /stats
    monitor-uri /healthy