Add PII correlation tool

Gemfile

@@ -2,6 +2,7 @@ source 'https://rubygems.org'
 ruby '>= 3.3', '< 3.5'
 
 # Essential gems: servers, adapters, Rails + Rails requirements
+gem 'bcrypt', '~> 3.1'
 gem 'coffee-rails', '~> 5.0.0'
 gem 'connection_pool', '< 3.0' # mperham/connection_pool#210
 gem 'counter_culture', '~> 3.2'
@@ -38,6 +39,7 @@ gem 'groupdate', '~> 6.1'
 
 # View stuff.
 gem 'diffy', '~> 3.4'
+gem 'ipaddress', '~> 0.8'
 gem 'jbuilder', '~> 2.11'
 gem 'rqrcode', '~> 2.1'
 gem 'will_paginate', '~> 3.3'

Gemfile.lock

@@ -181,6 +181,7 @@ GEM
       mini_magick (>= 4.9.5, < 6)
       ruby-vips (>= 2.0.17, < 3)
     io-console (0.8.2)
+    ipaddress (0.8.3)
     irb (1.17.0)
       pp (>= 0.6.0)
       prism (>= 1.3.0)
@@ -497,6 +498,7 @@ DEPENDENCIES
   aws-sdk-s3 (~> 1.208)
   aws-sdk-sns (~> 1.72)
   aws-ses-v4
+  bcrypt (~> 3.1)
   byebug (~> 11.1)
   capybara (~> 3.38)
   chartkick (~> 4.2)
@@ -512,6 +514,7 @@ DEPENDENCIES
   flamegraph (~> 0.9)
   groupdate (~> 6.1)
   image_processing (~> 1.12)
+  ipaddress (~> 0.8)
   jbuilder (~> 2.11)
   jquery-rails (~> 4.5.0)
   letter_opener_web (~> 2.0)

app/assets/javascripts/moderator.js

@@ -1,3 +1,3 @@
 $(() => {
   $('.js-convert-to-comment, .js-toggle-comments, .js-feature-post, .js-lock').on('ajax:success', (_ev) => {
     location.reload();
@@ -47,4 +47,14 @@
       checkbox.checked = action === 'all';
     });
   });
+
+  QPixel.DOM.addSelectorListener('submit', '#pii-correlation-form', async (ev) => {
+    ev.preventDefault();
+
+    const targetId = document.querySelector('input[name="target_id"]').value;
+    const resp = await QPixel.fetch(`${location.pathname}?format=template&target_id=${targetId}`);
+    const html = await resp.text();
+
+    document.querySelector('.js-correlation-container').innerHTML = html;
+  });
 });

app/controllers/moderator_controller.rb

@@ -89,6 +89,16 @@ def handle_spammy_users
     redirect_to mod_spammers_path
   end
 
+  def pii_correlation
+    @user = User.find(params[:id])
+    respond_to do |format|
+      format.html
+      format.template do
+        @target = User.find_by(id: params[:target_id])
+      end
+    end
+  end
+
   private
 
   def set_post

app/helpers/moderator_helper.rb

@@ -16,4 +16,29 @@ def text_bg(cls, content = nil, **opts, &block)
       tag.span content, class: ["has-background-color-#{cls}", opts[:class]].join(' ')
     end
   end
+
+  ##
+  # Split an IP address into an array of hashed octets (well, hexadecets for IPv6).
+  # @param ip [String] The IP address to process.
+  # @param salting_user [User] A user from which to source a salt for hashing. For hashes to be directly comparable, you
+  # must use the same user for each IP address you wish to compare, even if sourced from a different user.
+  # @return [[String, [String?]]] The IP address family, and an array of hashed octets.
+  def split_hash_ip(ip, salting_user)
+    begin
+      addr = IPAddress.parse(ip)
+    rescue ArgumentError
+      return ['', []]
+    end
+    splat = if addr.ipv6?
+              addr.hexs
+            else
+              addr.octets
+            end
+    salt = BCrypt::Password.new(salting_user.encrypted_password).salt
+    splat = splat.map { |p| Digest::SHA2.hexdigest(salt + p.to_s) }
+    [
+      addr.ipv6? ? 'IPv6' : 'IPv4',
+      splat
+    ]
+  end
 end

app/jobs/update_user_stats_job.rb

@@ -0,0 +1,11 @@
+class UpdateUserStatsJob < ApplicationJob
+  queue_as :default
+
+  def perform(*)
+    domains = User.all.select(:email)
+                  .group_by { |u| u.email&.split('@')[1] }
+                  .to_h { |d, u| [d, u.size] }
+                  .reject { |d, _u| d.include? 'localhost' }
+    Rails.cache.hmset('user_email_domains', domains)
+  end
+end

app/views/moderator/pii_correlation.html.erb

@@ -0,0 +1,20 @@
+<h2>PII correlation for <%= user_link @user %></h2>
+<p>
+  This tool displays correlations between personally-identifying information across user accounts. This includes email
+  address and IP addresses. Please select a user against whom to compare.
+</p>
+
+<noscript>JavaScript is required to use this tool.</noscript>
+<form id="pii-correlation-form" action="#" class="form-horizontal">
+  <div class="form-group-horizontal">
+    <div class="form-group">
+      <label for="target-id" class="form-element">Target user ID</label>
+      <input type="number" id="target-id" name="target_id" class="form-element" required />
+    </div>
+    <div class="actions">
+      <button type="submit" class="button is-primary is-filled">Compare</button>
+    </div>
+  </div>
+</form>
+
+<div class="js-correlation-container"></div>

app/views/moderator/pii_correlation.template.erb

@@ -0,0 +1,72 @@
+<h3>Comparing with <%= user_link @target %></h3>
+
+<h4>Email address</h4>
+<table class="table is-full-width is-striped is-with-hover">
+  <thead>
+  <tr>
+    <th></th>
+    <th><%= @user.rtl_safe_username %></th>
+    <th><%= @target.rtl_safe_username %></th>
+  </tr>
+  </thead>
+  <tbody>
+  <tr>
+    <td><strong>Handle</strong></td>
+    <% user_handle = Digest::SHA2.hexdigest(@user.email.split('@')[0]) %>
+    <% target_handle = Digest::SHA2.hexdigest(@target.email.split('@')[0]) %>
+    <td>
+      <code class="<%= 'has-background-color-red' if user_handle == target_handle %>">
+        <%= user_handle[0..7] %>
+      </code>
+    </td>
+    <td>
+      <code class="<%= 'has-background-color-red' if user_handle == target_handle %>">
+        <%= target_handle[0..7] %>
+      </code>
+    </td>
+  </tr>
+  <tr>
+    <td><strong>Domain</strong></td>
+    <td>
+      <code><%= Digest::SHA2.hexdigest(@user.email.split('@')[1])[0..7] %></code><br/>
+      <% domain_users = Rails.cache.hget 'user_email_domains', @user.email.split('@')[1] %>
+      <% if domain_users.nil? %>
+        (unknown number of users)
+      <% else %>
+        Used by <%= pluralize(domain_users, 'user') %>
+      <% end %>
+    </td>
+    <td>
+      <code><%= Digest::SHA2.hexdigest(@target.email.split('@')[1])[0..7] %></code><br/>
+      <% domain_users = Rails.cache.hget 'user_email_domains', @target.email.split('@')[1] %>
+      <% if domain_users.nil? %>
+        (unknown number of users)
+      <% else %>
+        Used by <%= pluralize(domain_users, 'user') %>
+      <% end %>
+    </td>
+  </tr>
+  </tbody>
+</table><br/>
+
+<h4>IP addresses</h4>
+
+<% user_current_family, user_current_ip = split_hash_ip(@user.current_sign_in_ip, @user) %>
+<% target_current_family, target_current_ip = split_hash_ip(@target.current_sign_in_ip, @user) %>
+<strong>Current sign-in</strong><br/>
+<table class="table is-striped is-with-hover">
+  <tr>
+    <td><%= @user.rtl_safe_username %></td>
+    <td>
+      <%= user_current_family %><br/>
+      <code><%= user_current_ip.map { |p| p[0..3] }.join(user_current_family == 'IPv4' ? '.' : ':') %></code>
+    </td>
+  </tr>
+  <tr>
+    <td><%= @target.rtl_safe_username %></td>
+    <td>
+      <%= target_current_family %><br/>
+      <code><%= target_current_ip.map { |p| p[0..3] }.join(target_current_family == 'IPv4' ? '.' : ':') %></code>
+    </td>
+  </tr>
+</table>

config/initializers/mime_types.rb

@@ -2,3 +2,5 @@
 
 # Add new mime types for use in respond_to blocks:
 # Mime::Type.register "text/richtext", :rtf
+
+Mime::Type.register('text/html+template', :template)

config/routes.rb

@@ -226,6 +226,7 @@
     get    '/:id/mod/activity-log',     to: 'users#full_log', as: :full_user_log
     post   '/:id/hellban',              to: 'admin#hellban', as: :hellban_user
     get    '/:id/avatar/:size',         to: 'users#avatar', as: :user_auto_avatar
+    get    '/:id/mod/pii',              to: 'moderator#pii_correlation', as: :mod_pii_correlation
   end
 
   post   'notifications/:id/read',         to: 'notifications#read', as: :read_notifications

config/schedule.rb

@@ -34,6 +34,10 @@
   runner 'scripts/run_new_thread_followers_cleanup.rb'
 end
 
+every 7.days, at: '05:00' do
+  runner 'scripts/run_user_stats.rb'
+end
+
 every 6.hours do
   runner 'scripts/recalc_abilities.rb'
 end

lib/namespaced_env_cache.rb

@@ -1,5 +1,10 @@
+require_relative 'redis_cache_hash_methods'
+
 module QPixel
   class NamespacedEnvCache < ActiveSupport::Cache::Store
+    include RedisCacheHashMethods
+    attr_reader :underlying
+
     def initialize(underlying)
       @underlying = underlying
       @getters = {}
@@ -144,6 +149,14 @@ def self.supports_cache_versioning?
       true
     end
 
+    def method_missing(name, *args, **opts, &block)
+      @underlying.send(name, *args, **opts, &block)
+    end
+
+    def respond_to_missing?(name, *)
+      @underlying.respond_to?(name)
+    end
+
     private
 
     # Raises an error if a given collection is not cacheable

lib/redis_cache_hash_methods.rb

@@ -0,0 +1,103 @@
+module QPixel
+  ##
+  # Class for inclusion in a +RedisCacheStore+ or +NamespacedEnvCache+ to add Redis hash methods.
+  # If the class is not a +RedisCacheStore+ or +NamespacedEnvCache+, these methods will still be added but will
+  # raise at runtime. The cache implementation must be using +ConnectionPool+.
+  module RedisCacheHashMethods
+    ##
+    # Set a hash value.
+    # @param hash_key [String] The name of the hash
+    # @param key [String] The key within the hash
+    # @param value [String] The key's value
+    # @return [Integer] The number of keys that were added to the hash
+    def hset(hash_key, key, value)
+      with_redis do |rd|
+        rd.hset hash_key, key, value
+      end
+    end
+
+    ##
+    # Set multiple hash values.
+    # @param hash_key [String] The name of the hash
+    # @param data [Hash] Keys and values to add to the hash
+    # @return [String] 'OK'
+    def hmset(hash_key, data)
+      with_redis do |rd|
+        rd.hmset hash_key, data.to_a.flatten
+      end
+    end
+
+    ##
+    # Get a hash value.
+    # @param hash_key [String] The name of the hash
+    # @param key [String] The key within the hash
+    # @return [String] The key's value
+    def hget(hash_key, key)
+      with_redis do |rd|
+        rd.hget hash_key, key
+      end
+    end
+
+    ##
+    # Get multiple hash values.
+    # @param hash_key [String] The name of the hash
+    # @param *keys [String] Keys within the hash to retrieve
+    # @return [Hash] Keys and values from the hash
+    def hmget(hash_key, *keys)
+      with_redis do |rd|
+        values = rd.hmget hash_key, *keys
+        keys.zip(values).to_h
+      end
+    end
+
+    ##
+    # Get all hash values.
+    # @param hash_key [String] The name of the hash
+    # @return [Hash] The hash's values
+    def hgetall(hash_key)
+      with_redis do |rd|
+        rd.hgetall hash_key
+      end
+    end
+
+    ##
+    # Delete a hash value, or the entire hash.
+    # @param hash_key [String] The name of the hash
+    # @param *keys [String] Keys within the hash to delete. If none are provided, the entire hash is deleted.
+    # @return [Integer] The number of keys that were removed from the hash
+    def hdel(hash_key, *keys)
+      with_redis do |rd|
+        if keys.size.zero?
+          rd.del hash_key
+        else
+          rd.hdel hash_key, *keys
+        end
+      end
+    end
+
+    private
+
+    ##
+    # Check a connection out of the connection pool and provides it to the block to run Redis commands.
+    # @yield [Redis::Client]
+    def with_redis
+      reject_unless_redis_cache!
+      redis_cache_store = ActiveSupport::Cache::RedisCacheStore
+      redis_cache = is_a?(redis_cache_store) ? self : underlying
+      redis_cache.redis.with do |rd|
+        yield rd
+      end
+    end
+
+    ##
+    # Raises an error unless the current class is a +RedisCacheStore+, or is a +NamespacedEnvCache+ that is backed by
+    # a +RedisCacheStore+.
+    # @raise [NotImplementedError]
+    def reject_unless_redis_cache!
+      redis_cache_store = ActiveSupport::Cache::RedisCacheStore
+      unless is_a?(redis_cache_store) || (respond_to?(:underlying) && underlying.is_a?(redis_cache_store))
+        raise NotImplementedError, "This cache implementation is not backed by Redis and cannot use Hash methods."
+      end
+    end
+  end
+end

scripts/run_user_stats.rb

@@ -0,0 +1 @@
+UpdateUserStatsJob.perform_later

test/jobs/update_user_stats_job_test.rb

@@ -0,0 +1,7 @@
+require "test_helper"
+
+class UpdateUserStatsJobTest < ActiveJob::TestCase
+  # test "the truth" do
+  #   assert true
+  # end
+end

test/lib/redis_cache_hash_methods_test.rb

@@ -0,0 +1,24 @@
+require 'test_helper'
+
+class RedisCacheHashMethodsTest < ActiveSupport::TestCase
+  test 'redis cache hash methods' do
+    assert_nil Rails.cache.read('test_hash')
+    assert_equal 1, Rails.cache.hset('test_hash', 'key', 'value')
+    assert_equal 'value', Rails.cache.hget('test_hash', 'key')
+    assert_equal 'OK', Rails.cache.hmset('test_hash', { 'key2' => 'value2', 'key3' => 'value3' })
+    assert_equal({ 'key' => 'value', 'key2' => 'value2', 'key3' => 'value3' },
+                 Rails.cache.hmget('test_hash', 'key', 'key2', 'key3'))
+    assert_equal({ 'key' => 'value', 'key2' => 'value2', 'key3' => 'value3' },
+                 Rails.cache.hgetall('test_hash'))
+    assert_equal 1, Rails.cache.hdel('test_hash', 'key3')
+    assert_equal 1, Rails.cache.hdel('test_hash')
+    assert_nil Rails.cache.read('test_hash')
+  end
+
+  test 'rejects calls on unimplemented caches' do
+    cache = QPixel::NamespacedEnvCache.new(ActiveSupport::Cache::MemoryStore.new)
+    assert_raises NotImplementedError do
+      cache.hgetall('test_hash')
+    end
+  end
+end