fix: Puell Multiple indicator range regression (#22)

api/coinsoto_api.py

@@ -4,7 +4,7 @@
 
 
 def cs_fetch(path: str, data_selector: str, col_name: str) -> pd.DataFrame:
-    response = HTTP.get(f'https://coinank.com/indicatorapi/{path}')
+    response = HTTP.get(f'https://api.coinank.com/indicatorapi/{path}')
     response.raise_for_status()
     data = response.json()['data']
 
@@ -15,12 +15,10 @@ def cs_fetch(path: str, data_selector: str, col_name: str) -> pd.DataFrame:
     data_y = data[data_selector]
     assert len(data_x) == len(data_y), f'{len(data_x)=} != {len(data_y)=}'
 
-    df = pd.DataFrame(
-        {
-            'Date': data_x[: len(data_y)],
-            col_name: data_y,
-        }
-    )
+    df = pd.DataFrame({
+        'Date': data_x[: len(data_y)],
+        col_name: data_y,
+    })
 
     df['Date'] = pd.to_datetime(df['Date'], unit='ms').dt.tz_localize(None)
 

fetch_bitcoin_data.py

@@ -1,43 +1,121 @@
+from itertools import count, pairwise
+
 import numpy as np
 import pandas as pd
 from filecache import filecache
 
 from utils import HTTP, mark_days_since, mark_highs_lows
 
+HALVING_INTERVAL = 210_000
+GENESIS_BLOCK_REWARD = 50.0
+BLOCKS_PER_DAY = 144
 
-@filecache(7200)  # 2 hours
-def fetch_bitcoin_data() -> pd.DataFrame:
+
+def fetch_block_halving():
     """
-    Fetches historical Bitcoin data into a DataFrame.
-    Very early data is discarded due to high volatility.
+    Fetch Bitcoin halving data by probing raw blocks at halving heights.
+    """
+    halving_data: list[tuple[pd.Timestamp, int, float]] = []
+
+    for halving_index in count():
+        block_height = halving_index * HALVING_INTERVAL
+        block_reward = GENESIS_BLOCK_REWARD / (2**halving_index)
+
+        response = HTTP.get(f'https://blockchain.info/rawblock/{block_height}')
+        if response.status_code == 404:
+            break
+
+        response.raise_for_status()
+        block_time = response.json()['time']
+        block_date = pd.to_datetime(block_time, unit='s').tz_localize(None).floor('d')
+        halving_data.append((block_date, block_height, block_reward))
+
+    return halving_data
+
+
+def fetch_blockchain_data() -> pd.DataFrame:
+    """
+    Fetches historical Bitcoin blockchain data from Blockchain.com API.
+    Uses miners-revenue chart for USD mining revenue.
+    Block heights and BTC generation are calculated from halving schedule.
 
     Returns:
-        DataFrame containing Bitcoin data.
+        DataFrame with Date, TotalBlocks, MinBlockID, MaxBlockID,
+        TotalGeneration, TotalGenerationUSD columns.
     """
-    print('📈 Requesting historical Bitcoin data…')
+    halving_data = fetch_block_halving()
 
+    # Fetch mining revenue from Blockchain.com
     response = HTTP.get(
-        'https://api.blockchair.com/bitcoin/blocks',
+        'https://api.blockchain.info/charts/miners-revenue',
         params={
-            'a': 'date,count(),min(id),max(id),sum(generation),sum(generation_usd)',
-            's': 'date(desc)',
+            'timespan': 'all',
+            'format': 'json',
+            'sampled': 'false',
         },
     )
+
+    # Create DataFrame from mining revenue data
     response.raise_for_status()
-    response_json = response.json()
+    df = pd.DataFrame(response.json()['values'])
+    df.columns = ['DateTimestamp', 'TotalGenerationUSD']
+    df['Date'] = pd.to_datetime(df['DateTimestamp'], unit='s').dt.floor('d')
+
+    # Calculate approximate block height for each day using linear interpolation
+    # between known halving points
+    def estimate_block_height(date: pd.Timestamp):
+        # Find the halving period this date falls into
+        for (start_date, start_height, _), (end_date, end_height, _) in pairwise(halving_data):
+            if start_date <= date < end_date:
+                # Linear interpolation within this halving period
+                total_days = (end_date - start_date).days
+                days_elapsed = (date - start_date).days
+                height = start_height + (end_height - start_height) * days_elapsed / total_days
+                return int(height)
+
+        # After the last known halving, extrapolate
+        (last_date, last_height, _) = halving_data[-1]
+        days_since = (date - last_date).days
+        return int(last_height + days_since * BLOCKS_PER_DAY)
+
+    def get_block_reward(block_height):
+        """Get block reward for a given block height."""
+        halvings = block_height // HALVING_INTERVAL
+        return GENESIS_BLOCK_REWARD / (2**halvings)
+
+    # Calculate block data for each day
+    df['MaxBlockID'] = df['Date'].apply(estimate_block_height)
+    df['MinBlockID'] = df['MaxBlockID'].shift(1, fill_value=0)
+    df['TotalBlocks'] = df['MaxBlockID'] - df['MinBlockID']
+
+    # Calculate BTC generation based on block reward
+    df['BlockReward'] = df['MaxBlockID'].apply(get_block_reward)
+    df['TotalGeneration'] = df['TotalBlocks'] * df['BlockReward'] * 1e8  # Convert to satoshis
+
+    # Select and order columns to match original format
+    df = df[['Date', 'TotalBlocks', 'MinBlockID', 'MaxBlockID', 'TotalGeneration', 'TotalGenerationUSD']]
+    df = df.sort_values('Date').reset_index(drop=True)
+
+    # Add halving markers
+    df['Halving'] = 0
+    for _, block_height, _ in halving_data[1:]:
+        df.loc[(df['MinBlockID'] < block_height) & (df['MaxBlockID'] >= block_height), 'Halving'] = 1
+
+    return df
 
-    df = pd.DataFrame(response_json['data'][::-1])
-    df.rename(
-        columns={
-            'date': 'Date',
-            'count()': 'TotalBlocks',
-            'min(id)': 'MinBlockID',
-            'max(id)': 'MaxBlockID',
-            'sum(generation)': 'TotalGeneration',
-            'sum(generation_usd)': 'TotalGenerationUSD',
-        },
-        inplace=True,
-    )
+
+@filecache(7200)  # 2 hours
+def fetch_bitcoin_data() -> pd.DataFrame:
+    """
+    Fetches historical Bitcoin data into a DataFrame.
+    Very early data is discarded due to high volatility.
+
+    Returns:
+        DataFrame containing Bitcoin data.
+    """
+    print('📈 Requesting historical Bitcoin data…')
+
+    df = fetch_blockchain_data()
 
     df['Date'] = pd.to_datetime(df['Date'])
     df['TotalGeneration'] /= 1e8
@@ -57,7 +135,6 @@ def fetch_bitcoin_data() -> pd.DataFrame:
     df.reset_index(drop=True, inplace=True)
 
     df = fix_current_day_data(df)
-    df = add_block_halving_data(df)
     df = mark_highs_lows(df, 'Price', False, round(365 * 2), 180)
 
     # move 2021' peak to the first price peak
@@ -82,12 +159,10 @@ def fetch_price_data() -> pd.DataFrame:
     response_x = [float(k) for k in response_json['data']['points']]
     response_y = [value['v'][0] for value in response_json['data']['points'].values()]
 
-    df = pd.DataFrame(
-        {
-            'Date': response_x,
-            'Price': response_y,
-        }
-    )
+    df = pd.DataFrame({
+        'Date': response_x,
+        'Price': response_y,
+    })
     df['Date'] = pd.to_datetime(df['Date'], unit='s').dt.tz_localize(None).dt.floor('d')
     df.sort_values(by='Date', inplace=True)
     df.drop_duplicates('Date', keep='last', inplace=True)
@@ -98,41 +173,10 @@ def fetch_price_data() -> pd.DataFrame:
 def fix_current_day_data(df: pd.DataFrame) -> pd.DataFrame:
     row = df.iloc[-1].copy()
 
-    target_total_blocks = 24 * 6
-    target_scale = target_total_blocks / row['TotalBlocks']
+    target_scale = BLOCKS_PER_DAY / row['TotalBlocks']
 
     for col_name in ['TotalBlocks', 'TotalGeneration', 'TotalGenerationUSD']:
         row[col_name] *= target_scale
 
     df.iloc[-1] = row
     return df
-
-
-def add_block_halving_data(df: pd.DataFrame) -> pd.DataFrame:
-    reward_halving_every = 210000
-    current_block_halving_id = reward_halving_every
-    current_block_production = 50
-    df['Halving'] = 0
-    df['NextHalvingBlock'] = current_block_halving_id
-
-    while True:
-        df.loc[
-            (current_block_halving_id - reward_halving_every) <= df['MaxBlockID'],
-            'BlockGeneration',
-        ] = current_block_production
-
-        block_halving_row = df[
-            (df['MinBlockID'] <= current_block_halving_id) & (df['MaxBlockID'] >= current_block_halving_id)
-        ].squeeze()
-
-        if block_halving_row.shape[0] == 0:
-            break
-
-        current_block_halving_id += reward_halving_every
-        current_block_production /= 2
-        df.loc[block_halving_row.name, 'Halving'] = 1
-        df.loc[df.index > block_halving_row.name, 'NextHalvingBlock'] = current_block_halving_id
-
-    df['DaysToHalving'] = pd.to_timedelta((df['NextHalvingBlock'] - df['MaxBlockID']) / (24 * 6), unit='D')
-    df['NextHalvingDate'] = df['Date'] + df['DaysToHalving']
-    return df

metrics/pi_cycle.py

@@ -63,4 +63,8 @@ def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series:
         sns.lineplot(data=df, x='Date', y='PiCycleIndexNoNa', ax=ax[0])
         add_common_markers(df, ax[0])
 
+        sns.lineplot(data=df, x='Date', y='PiCycleDiff', ax=ax[1])
+        sns.lineplot(data=df, x='Date', y='PiCycleDiffThreshold', ax=ax[1], linestyle='--')
+        add_common_markers(df, ax[1], price_line=False)
+
         return df['PiCycleIndex']

metrics/puell_multiple.py

@@ -4,9 +4,8 @@
 from matplotlib.axes import Axes
 from sklearn.linear_model import LinearRegression
 
-from api.coinsoto_api import cs_fetch
 from metrics.base_metric import BaseMetric
-from utils import add_common_markers
+from utils import add_common_markers, mark_highs_lows
 
 
 class PuellMetric(BaseMetric):
@@ -19,19 +18,16 @@ def description(self) -> str:
         return 'Puell Multiple'
 
     def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series:
-        df = df.merge(
-            cs_fetch(
-                path='getPuellMultiple',
-                data_selector='puellMultiplList',
-                col_name='Puell',
-            ),
-            on='Date',
-            how='left',
-        )
+        # Calculate Puell Multiple locally from mining revenue data
+        # Puell = daily_mining_revenue / 365-day_MA_of_mining_revenue
+        # TotalGenerationUSD contains daily mining revenue in USD from Blockchain.com
+        df['Puell'] = df['TotalGenerationUSD'] / df['TotalGenerationUSD'].rolling(window=365, min_periods=1).mean()
         df['Puell'] = df['Puell'].ffill()
         df['PuellLog'] = np.log(df['Puell'])
 
-        high_rows = df.loc[df['PriceHigh'] == 1]
+        df = mark_highs_lows(df, 'PuellLog', True, round(365 * 2), 365)
+        high_rows = df.loc[(df['PuellLogHigh'] == 1) & (df.index > 365)]
+
         high_x = high_rows.index.values.reshape(-1, 1)
         high_y = high_rows['PuellLog'].values.reshape(-1, 1)
 
@@ -43,7 +39,9 @@ def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series:
 
         lin_model = LinearRegression()
         lin_model.fit(high_x, high_y)
-        df['PuellLogHighModel'] = lin_model.predict(x)
+        predictions = lin_model.predict(x)
+        min_peak = high_y.min()
+        df['PuellLogHighModel'] = np.maximum(predictions, min_peak)
 
         # lin_model.fit(low_x, low_y)
         # df['PuellLogLowModel'] = lin_model.predict(x)

metrics/two_year_moving_average.py

@@ -4,7 +4,6 @@
 from matplotlib.axes import Axes
 from sklearn.linear_model import LinearRegression
 
-from api.coinsoto_api import cs_fetch
 from metrics.base_metric import BaseMetric
 from utils import add_common_markers
 
@@ -19,15 +18,9 @@ def description(self) -> str:
         return '2 Year Moving Average'
 
     def _calculate(self, df: pd.DataFrame, ax: list[Axes]) -> pd.Series:
-        df = df.merge(
-            cs_fetch(
-                path='getBtcMultiplier',
-                data_selector='mA730List',
-                col_name='2YMA',
-            ),
-            on='Date',
-            how='left',
-        )
+        # Calculate 2-year (730-day) moving average locally from price data
+        # No external API needed - we have the price data already
+        df['2YMA'] = df['Price'].rolling(window=730, min_periods=1).mean()
         df['2YMA'] = df['2YMA'].ffill()
         df['2YMALog'] = np.log(df['2YMA'])
         df['2YMALogDiff'] = df['PriceLog'] - df['2YMALog']

utils.py

@@ -237,4 +237,4 @@ async def send_error_notification(exception: Exception) -> bool:
             f'<pre>{"".join(traceback.format_exception(exception))}</pre>',
             parse_mode='HTML',
         )
-    return True
+    return True