Implement ELECTRIC_IDLE_WAL_SIZE_CHECK_PERIOD and ELECTRIC_IDLE_WAL_SIZE_THRESHOLD

Author: alco

packages/sync-service/config/runtime.exs

@@ -272,7 +272,14 @@ config :electric,
       "ELECTRIC_REPLICATION_IDLE_TIMEOUT",
       &Electric.Config.parse_human_readable_time!/1,
       nil
-    )
+    ),
+  idle_wal_size_check_period:
+    env!(
+      "ELECTRIC_IDLE_WAL_SIZE_CHECK_PERIOD",
+      &Electric.Config.parse_human_readable_time!/1,
+      nil
+    ),
+  idle_wal_size_threshold: env!("ELECTRIC_IDLE_WAL_SIZE_THRESHOLD", :integer, nil)
 
 if Electric.telemetry_enabled?() do
   config :sentry,

packages/sync-service/lib/electric/application.ex

@@ -144,7 +144,9 @@ defmodule Electric.Application do
         publication_refresh_period: get_env(opts, :publication_refresh_period),
         schema_reconciler_period: get_env(opts, :schema_reconciler_period),
         cleanup_interval_ms: get_env(opts, :cleanup_interval_ms),
-        shape_hibernate_after: get_env(opts, :shape_hibernate_after)
+        shape_hibernate_after: get_env(opts, :shape_hibernate_after),
+        idle_wal_size_check_period: get_env(opts, :idle_wal_size_check_period),
+        idle_wal_size_threshold: get_env(opts, :idle_wal_size_threshold)
       ],
       manual_table_publishing?: get_env(opts, :manual_table_publishing?)
     )

packages/sync-service/lib/electric/config.ex

@@ -50,6 +50,13 @@ defmodule Electric.Config do
     max_txn_size: 250 * 1024 * 1024,
     # Scaling down on idle is disabled by default
     replication_idle_timeout: 0,
+    # If the database provider scales down after 5 min and provided that the
+    # replication_idle_timeout is about a minute or less, checking WAL size once an hour
+    # ends up using about 10% of the compute on an otherwise idle database.
+    idle_wal_size_check_period: 3_600_000,
+    # We want to wake up and process any transactions that have accumulated in the WAL, hence
+    # the low threshold.
+    idle_wal_size_threshold: 1_000,
     manual_table_publishing?: false,
     ## HTTP API
     # set enable_http_api: false to turn off the HTTP server totally
@@ -418,7 +425,7 @@ defmodule Electric.Config do
     end
   end
 
-  @time_units ~w[ms msec s sec m min]
+  @time_units ~w[ms msec s sec m min h hr]
 
   @spec parse_human_readable_time(binary | nil) :: {:ok, pos_integer} | {:error, binary}
 
@@ -437,6 +444,7 @@ defmodule Electric.Config do
   defp time_multiplier(millisecond) when millisecond in ["ms", "msec"], do: 1
   defp time_multiplier(second) when second in ["s", "sec"], do: 1000
   defp time_multiplier(minute) when minute in ["m", "min"], do: 1000 * 60
+  defp time_multiplier(hour) when hour in ["h", "hr"], do: 1000 * 60 * 60
 
   def parse_human_readable_time!(str) do
     case parse_human_readable_time(str) do

packages/sync-service/lib/electric/stack_supervisor.ex

@@ -118,6 +118,14 @@ defmodule Electric.StackSupervisor do
                      shape_hibernate_after: [
                        type: :integer,
                        default: Electric.Config.default(:shape_hibernate_after)
+                     ],
+                     idle_wal_size_check_period: [
+                       type: :integer,
+                       default: Electric.Config.default(:idle_wal_size_check_period)
+                     ],
+                     idle_wal_size_threshold: [
+                       type: :integer,
+                       default: Electric.Config.default(:idle_wal_size_threshold)
                      ]
                    ]
                  ],