Add data dumps

.github/workflows/ci-cd.yml

@@ -62,7 +62,7 @@ jobs:
 
     services:
       mysql: &db-service
-        image: mysql:8.0
+        image: mysql:8.4.2 # please keep the image in sync with Dockerfile.db
         env:
           MYSQL_ROOT_HOST: '%'
           MYSQL_ROOT_PASSWORD: 'root'

INSTALLATION.md

@@ -131,6 +131,7 @@ the MySQL server with `sudo mysql -u root` and create a new database user for QP
 CREATE USER qpixel@localhost IDENTIFIED BY 'choose_a_password_here';
 GRANT ALL ON qpixel_dev.* TO qpixel@localhost;
 GRANT ALL ON qpixel_test.* TO qpixel@localhost;
+GRANT ALL ON qpixel_dump.* TO qpixel@localhost;
 GRANT ALL ON qpixel.* TO qpixel@localhost;
 ```
 

app/controllers/dumps_controller.rb

@@ -0,0 +1,8 @@
+class DumpsController < ApplicationController
+  before_action :authenticate_user!
+
+  def index
+    @latest = Dump.automatic.last
+    @others = Dump.manual
+  end
+end

app/helpers/dumps_helper.rb

@@ -0,0 +1,2 @@
+module DumpsHelper
+end

app/jobs/data_dump_job.rb

@@ -0,0 +1,92 @@
+class DataDumpJob < ApplicationJob
+  queue_as :default
+
+  def perform(drop_db_after: true)
+    permitted = YAML.safe_load_file(Rails.root.join('db/scripts/dump_permitted_columns.yml'))
+    logger.info "Found #{permitted&.size} tables to dump."
+
+    begin
+      exec('SET FOREIGN_KEY_CHECKS = 0;')
+      exec('DROP DATABASE IF EXISTS qpixel_dump;')
+      exec('CREATE DATABASE qpixel_dump;')
+
+      @db_creds = Rails.configuration.database_configuration[Rails.env]
+      @username = @db_creds['username']
+      @password = @db_creds['password']
+      @database = @db_creds['database']
+      @port = @db_creds['port']
+      @host = @db_creds['host']
+
+      mysqldump_command = build_command('mysqldump', '-h', @host, '-u', @username, "-p#{@password}", '-d', @database,
+                                        '--no-tablespaces', "--port=#{@port}", ssl_state)
+      mysql_command = build_command('mysql', '-h', @host, '-u', @username, "-p#{@password}", "--port=#{@port}",
+                                    '-D', 'qpixel_dump', ssl_state)
+      logger.debug 'Running system command:'
+      logger.debug "#{mysqldump_command} | #{mysql_command}"
+      copy_success = system("#{mysqldump_command} | #{mysql_command}")
+
+      unless copy_success
+        logger.fatal "Couldn't replicate database: nonzero exit code"
+        return
+      end
+
+      logger.info 'Copied database structure.'
+
+      permitted&.each do |table, data|
+        migrate_table(table, data)
+      end
+
+      logger.info 'Migrated data.'
+
+      file_path = Rails.root.join('tmp/qpixel_export.sql')
+      export_cmd = build_command('mysqldump', '-h', @host, '-u', @username, "-p#{@password}", "--port=#{@port}",
+                                 'qpixel_dump', '--no-tablespaces', ssl_state, '>', file_path)
+      logger.debug 'Running system command:'
+      logger.debug export_cmd
+      export_success = system(export_cmd)
+
+      unless export_success
+        logger.fatal "Couldn't export database: nonzero exit code"
+        return
+      end
+
+      logger.info 'Exported database.'
+
+      dump = Dump.create(title: "Data Dump #{Time.now.strftime('%Y-%m-%d')}",
+                         comment: "Automatically generated data dump as of #{Time.now.strftime('%Y-%m-%d %H:%M:%S')}.",
+                         file: File.open(file_path),
+                         automatic: true)
+      Dump.where(automatic: true).where.not(id: dump.id).destroy_all
+    rescue ActiveRecord::ConnectionFailed
+      logger.fatal "Couldn't connect to database. Have you run `GRANT ALL ON qpixel_dump.*` for your DB user?"
+    ensure
+      exec('SET FOREIGN_KEY_CHECKS = 1;')
+      if drop_db_after
+        exec('DROP DATABASE qpixel_dump;')
+      end
+    end
+  end
+
+  private
+
+  def migrate_table(table, data)
+    columns = data['columns']
+    query = data['query']
+    select = "(SELECT #{columns.map { |c| "`#{table}`.`#{c}`" }.join(', ')} FROM #{@database}.#{table} #{query})"
+    full_query = "INSERT INTO qpixel_dump.`#{table}` (#{columns.map { |c| "`#{c}`" }.join(', ')}) #{select}"
+    logger.debug full_query
+    exec(full_query)
+  end
+
+  def exec(sql)
+    ApplicationRecord.connection.execute(sql)
+  end
+
+  def build_command(cmd, *args)
+    "#{cmd} #{args.compact_blank.join(' ')}"
+  end
+
+  def ssl_state
+    '--skip-ssl' if Rails.env.development? || Rails.env.test?
+  end
+end

app/models/dump.rb

@@ -0,0 +1,14 @@
+class Dump < ApplicationRecord
+  has_one_attached :file
+
+  before_destroy :delete_file
+
+  scope :automatic, -> { where(automatic: true) }
+  scope :manual, -> { where(automatic: false) }
+
+  private
+
+  def delete_file
+    file.purge
+  end
+end

app/views/dumps/index.html.erb

@@ -0,0 +1,70 @@
+<h1>Data Dumps</h1>
+
+<p>
+  Data from all <%= SiteSetting['NetworkName'] %> communities is made available in database format for download here.
+  This data is a weekly export of the entire database, minus any personally identifiable information, moderation data,
+  and some other sensitive information such as who cast votes.
+</p>
+
+<div class="notice is-info has-color-tertiary-900">
+  <p>
+    <i class="fas fa-balance-scale"></i>
+    <strong>Licensing</strong>
+  </p>
+  <p>
+    This data is provided free of charge as part of our contribution to the commons. If you use post content, you must
+    still abide by the terms of the licenses set by the author of each post.
+  </p>
+</div>
+
+<h2>Latest data dump</h2>
+<% if @latest.nil? %>
+  <p>No data dumps available yet. Check back next week.</p>
+<% else %>
+  <table class="table is-with-hover is-full-width">
+    <thead>
+    <tr>
+      <th></th>
+      <th>Filename</th>
+      <th>Created</th>
+      <th>Download</th>
+    </tr>
+    </thead>
+    <tbody>
+    <tr>
+      <td><%= @latest.title %></td>
+      <td><%= @latest.file.filename %></td>
+      <td><%= @latest.created_at.strftime('%Y-%m-%d') %></td>
+      <td><%= link_to 'Download', rails_blob_path(@latest.file, disposition: 'attachment') %></td>
+    </tr>
+    </tbody>
+  </table>
+<% end %>
+
+<% if @others.any? %>
+  <h2>Other data dumps</h2>
+  <table class="table is-with-hover is-full-width">
+    <thead>
+      <tr>
+        <th></th>
+        <th>Created</th>
+        <th>Download</th>
+      </tr>
+    </thead>
+    <tbody>
+      <% @others.each do |dump| %>
+        <tr>
+          <td><%= dump.title %></td>
+          <td><%= dump.created_at.strftime('%Y-%m-%d') %></td>
+          <td>
+            <% if dump.file.attached? %>
+              <%= link_to 'Download', rails_blob_path(@latest.file, disposition: 'attachment') %>
+            <% elsif dump.link.present? %>
+              <%= link_to 'View', dump.link %>
+            <% end %>
+          </td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+<% end %>

app/views/layouts/_footer.html.erb

@@ -7,6 +7,7 @@
           <li><%= link_to 'About Us', '/policy/network-faq' %></li>
           <li><%= link_to 'Privacy & Safety Center', safety_center_url %></li>
           <li><%= link_to 'Report harmful content', new_complaint_path %></li>
+          <li><%= link_to 'Data dumps', data_dumps_path %></li>
         </ul>
       </div>
       <div class="grid--cell is-6 is-12-md is-12-sm">

config/environments/test.rb

@@ -13,7 +13,7 @@
   config.cache_classes = false
   config.action_view.cache_template_loading = true
 
-  config.log_level = :info
+  config.log_level = ENV['LOG_LEVEL'] || :info
 
   # Do not eager load code on boot. This avoids loading your whole application
   # just for the purpose of running a single test. If you are using a tool that

config/routes.rb

@@ -407,4 +407,8 @@
   scope 'network' do
     root                                   to: 'fake_community#communities', as: :fc_communities
   end
+
+  scope 'data' do
+    root                                   to: 'dumps#index', as: :data_dumps
+  end
 end

config/schedule.rb

@@ -34,6 +34,10 @@
   runner 'scripts/run_new_thread_followers_cleanup.rb'
 end
 
+every 7.days, at: '04:00' do
+  runner 'scripts/data_dump.rb'
+end
+
 every 6.hours do
   runner 'scripts/recalc_abilities.rb'
 end

db/migrate/20260120032240_create_dumps.rb

@@ -0,0 +1,10 @@
+class CreateDumps < ActiveRecord::Migration[7.2]
+  def change
+    create_table :dumps do |t|
+      t.string :title, null: false
+      t.string :comment
+
+      t.timestamps
+    end
+  end
+end

db/migrate/20260513135320_add_defaults_for_data_dumps.rb

@@ -0,0 +1,12 @@
+class AddDefaultsForDataDumps < ActiveRecord::Migration[7.2]
+  def change
+    # Add a bunch of default values for NOT NULL columns that don't already have them, so that data dumps don't break
+    # when the data in these columns isn't included. Only applies to NOT NULL columns that are NOT included in the dump,
+    # and do not already have a default value.
+    change_column_default :filters, :user_id, -1
+    change_column_default :flags, :escalated, false
+    change_column_default :users, :sign_in_count, 0
+    change_column_default :users, :failed_attempts, 0
+    change_column_default :users, :deleted, false
+  end
+end

db/migrate/20260513160917_add_automatic_to_dumps.rb

@@ -0,0 +1,5 @@
+class AddAutomaticToDumps < ActiveRecord::Migration[7.2]
+  def change
+    add_column :dumps, :automatic, :boolean, null: false, default: false
+  end
+end

db/migrate/20260513185013_add_link_to_dumps.rb

@@ -0,0 +1,5 @@
+class AddLinkToDumps < ActiveRecord::Migration[7.2]
+  def change
+    add_column :dumps, :link, :string
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.2].define(version: 2026_03_22_151439) do
+ActiveRecord::Schema[7.2].define(version: 2026_05_13_185013) do
   create_table "abilities", charset: "utf8mb4", collation: "utf8mb4_0900_ai_ci", force: :cascade do |t|
     t.bigint "community_id"
     t.string "name"
@@ -268,6 +268,15 @@
     t.index ["user_id"], name: "index_complaints_on_user_id"
   end
 
+  create_table "dumps", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
+    t.string "title", null: false
+    t.string "comment"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.boolean "automatic", default: false, null: false
+    t.string "link"
+  end
+
   create_table "email_logs", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
     t.string "log_type"
     t.string "destination"
@@ -294,7 +303,7 @@
   end
 
   create_table "filters", charset: "utf8mb4", collation: "utf8mb4_unicode_ci", force: :cascade do |t|
-    t.bigint "user_id", null: false
+    t.bigint "user_id", default: -1, null: false
     t.string "name", null: false
     t.float "min_score"
     t.float "max_score"