diff --git a/CHANGELOG.md b/CHANGELOG.md index 486797d..18360b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Cross-base requests for pegged quotes are now anchored through the peg's base. For example, `?base=EUR"es=AED` previously returned the blended `EUR/AED` rate from providers; it now returns `blended(EUR/USD) × peg(USD/AED)`. The numerical difference is small (basis points) but removes spurious provider-disagreement noise on quantities that are mathematically pinned by the issuing authority. (#323) - Pegs are now treated as a source of rate data alongside providers. When `?providers=` filters the source set, pegs are excluded along with all other unlisted sources. Requests like `?base=BMD&providers=ecb` (where ECB does not publish BMD) now return empty rather than synthesizing rates from the peg. Default behavior (no `providers=`) is unchanged. (#323) - IMF Special Drawing Rights (XDR) is no longer filtered out of provider backfills. Several providers (NB, SBI, BCRA, etc.) publish XDR rates that were silently dropped; they now flow through like any other ISO 4217 quote. Re-backfill from `coverage_start` to ingest previously-dropped rows. (#333) +- `/v2/rates` responses now stamp every row with its actual observation date rather than a flattened reporting date. On the latest and single-date paths, a pair carried forward from an earlier date now reports that earlier date. Range queries no longer carry forward at all: each pair appears on the days it actually published, with its own date. Pairs that didn't publish during the range are absent rather than projected. Mixed-cadence providers (e.g. weekly publishers) appear at their actual publication frequency. (#338) - Deutsche Bundesbank (BBK) as historical provider — daily pre-euro Frankfurt fixings for 18 currencies, 1948-06-21 through 1998-12-30 - Bank of Russia (CBR) precious-metal reference prices — daily XAU, XAG, XPT and XPD against RUB, available from 2008-07-01. CBR is the first source for platinum and palladium beyond the National Bank of Ukraine. - National Bank of Moldova (NBM) precious-metal reference prices — daily XAU and XAG against MDL, available from 2012-01-02. Brings XAG to four providers (CBA, NBU, CBR, NBM), clearing the consensus threshold for silver. diff --git a/lib/carry_forward.rb b/lib/carry_forward.rb index 747367e..3c500ef 100644 --- a/lib/carry_forward.rb +++ b/lib/carry_forward.rb @@ -1,51 +1,46 @@ # frozen_string_literal: true -# Carries forward each provider's most recent rate within a lookback window. Used for single-date -# queries (latest) and range query enrichment. -module CarryForward - LATEST_LOOKBACK_DAYS = 14 - RANGE_LOOKBACK_DAYS = 5 +# Produces a snapshot of rates as of a target date by carrying forward each provider's most recent +# rate within a lookback window. Used for single-date and latest queries; range queries do not +# carry forward. +class CarryForward + LOOKBACK_DAYS = 14 class << self - # Returns the most recent rate per (provider, base, quote) on or before the target date, within - # the lookback window. - def latest(rows, date:, lookback: LATEST_LOOKBACK_DAYS) - cutoff = date - lookback - best = {} + def apply(rows, date:, lookback: LOOKBACK_DAYS) + new(rows, date:, lookback:).apply + end + end - rows.each do |row| - d = row[:date] - next unless d&.between?(cutoff, date) + attr_reader :rows, :date, :lookback - key = [row[:provider], row[:base], row[:quote]] - best[key] = row if !best[key] || d > best[key][:date] - end + def initialize(rows, date:, lookback:) + @rows = rows + @date = date + @lookback = lookback + end - best.values + def apply + best = {} + eligible_rows.each do |row| + key = key_for(row) + best[key] = row if !best[key] || row[:date] > best[key][:date] end - # Enriches each date in the target range with carried-forward rates. Returns { date => [rows] } - # where each date's rows include both same-day rates and each provider's most recent rate within - # the lookback window. Carried-forward rows keep their original dates so WeightedAverage can - # discount them by staleness. - def enrich(rows, range:, lookback: RANGE_LOOKBACK_DAYS) - by_date = rows.group_by { |r| r[:date] } - target_dates = by_date.keys.select { |d| range.cover?(d) }.sort - - index = {} - rows.each do |row| - key = [row[:provider], row[:base], row[:quote]] - (index[key] ||= []) << row - end - index.each_value { |v| v.sort_by! { |r| r[:date] }.reverse! } - - target_dates.to_h do |date| - cutoff = date - lookback - group = index.filter_map do |_, dated_rows| - dated_rows.find { |r| r[:date].between?(cutoff, date) } - end - [date, group] - end - end + best.values + end + + private + + def cutoff + @cutoff ||= date - lookback + end + + def eligible_rows + rows.select { |r| r[:date].between?(cutoff, date) } + end + + def key_for(row) + [row[:provider], row[:base], row[:quote]] end end diff --git a/lib/versions/v1/currency_names.rb b/lib/versions/v1/currency_names.rb index 549afc6..e47f284 100644 --- a/lib/versions/v1/currency_names.rb +++ b/lib/versions/v1/currency_names.rb @@ -36,7 +36,7 @@ def find_currencies today = Date.today rows = Rate.where(provider: "ECB").where(date: (today - 14)..today).naked.all - CarryForward.latest(rows, date: today) + CarryForward.apply(rows, date: today) end end end diff --git a/lib/versions/v1/quote/end_of_day.rb b/lib/versions/v1/quote/end_of_day.rb index 8a2d467..50da9f0 100644 --- a/lib/versions/v1/quote/end_of_day.rb +++ b/lib/versions/v1/quote/end_of_day.rb @@ -32,7 +32,7 @@ def fetch_data scope = Rate.where(provider: "ECB").where(date: (date_val - 14)..date_val) scope = scope.only(*(symbols + [base])) if symbols - CarryForward.latest(scope.naked.all, date: date_val) + CarryForward.apply(scope.naked.all, date: date_val) end end end diff --git a/lib/versions/v2/rate_query.rb b/lib/versions/v2/rate_query.rb index 32c1964..5d808e3 100644 --- a/lib/versions/v2/rate_query.rb +++ b/lib/versions/v2/rate_query.rb @@ -21,6 +21,8 @@ class RateQuery class ValidationError < StandardError; end + ALLOWED_EXPANSIONS = ["providers"].freeze + ALLOWED_PARAMS = ["base", "quotes", "providers", "date", "from", "to", "group", "expand"].freeze CHUNK_MONTHS = { "week" => 21, "month" => 84 }.freeze DEFAULT_CHUNK_MONTHS = 3 @@ -41,23 +43,15 @@ def each(&block) ds = range_dataset date_col = ds.model.date_column - if rollup? - rows = ds.between(chunk_range).all - normalize_dates!(rows, date_col) if date_col != :date - rows.group_by { |r| r[:date] }.each do |_, group_rows| - emit_blended(group_rows, &block) - end - else - expanded = (chunk_range.begin - CarryForward::RANGE_LOOKBACK_DAYS)..chunk_range.end - rows = ds.between(expanded).naked.all - CarryForward.enrich(rows, range: chunk_range).each do |target_date, group_rows| - emit_blended(group_rows, target_date:, &block) - end + rows = ds.between(chunk_range).all + normalize_dates!(rows, date_col) if date_col != :date + rows.group_by { |r| r[:date] }.each do |_, group_rows| + emit_blended(group_rows, &block) end end else - window = raw_dataset.where(date: (date_scope - CarryForward::LATEST_LOOKBACK_DAYS)..date_scope) - rows = CarryForward.latest(window.naked.all, date: date_scope) + window = raw_dataset.where(date: (date_scope - CarryForward::LOOKBACK_DAYS)..date_scope) + rows = CarryForward.apply(window.naked.all, date: date_scope) emit_blended(rows, &block) end end @@ -81,7 +75,7 @@ def max_date if date_scope.is_a?(Range) ds.where(date: date_scope).max(:date) else - ds.where(date: (date_scope - CarryForward::LATEST_LOOKBACK_DAYS)..date_scope).max(:date) + ds.where(date: (date_scope - CarryForward::LOOKBACK_DAYS)..date_scope).max(:date) end end @@ -165,9 +159,6 @@ def parse_date(value) nil end - ALLOWED_PARAMS = ["base", "quotes", "providers", "date", "from", "to", "group", "expand"].freeze - ALLOWED_EXPANSIONS = ["providers"].freeze - def validate! validate_params! validate_dates! @@ -218,17 +209,15 @@ def date_scope end end - def emit_blended(rows, target_date: nil, &block) + def emit_blended(rows, &block) blended = Blender.new(rows, base: providers ? base : effective_base).blend blended = PegAnchor.apply(blended, base: base, base_peg: base_peg) unless providers return if blended.empty? - output_date = (target_date || blended.map { |r| r[:date] }.max)&.to_s - records = blended.filter_map do |r| next if quotes && !quotes.include?(r[:quote]) - record = { date: output_date, base: r[:base], quote: r[:quote], rate: round(r[:rate]) } + record = { date: r[:date].to_s, base: r[:base], quote: r[:quote], rate: round(r[:rate]) } record[:providers] = r[:providers] if expand_providers? && r[:providers] record end diff --git a/spec/carry_forward_spec.rb b/spec/carry_forward_spec.rb index 7af6adc..6d92bd5 100644 --- a/spec/carry_forward_spec.rb +++ b/spec/carry_forward_spec.rb @@ -4,7 +4,7 @@ require "carry_forward" describe CarryForward do - describe ".latest" do + describe ".apply" do it "returns the most recent rate per provider/base/quote" do rows = [ { date: Date.new(2024, 1, 5), provider: "ECB", base: "EUR", quote: "USD", rate: 1.08 }, @@ -12,7 +12,7 @@ { date: Date.new(2024, 1, 5), provider: "BOC", base: "CAD", quote: "USD", rate: 0.74 }, ] - result = CarryForward.latest(rows, date: Date.new(2024, 1, 6)) + result = CarryForward.apply(rows, date: Date.new(2024, 1, 6)) _(result.size).must_equal(2) ecb = result.find { |r| r[:provider] == "ECB" } @@ -26,7 +26,7 @@ { date: Date.new(2024, 1, 1), provider: "ECB", base: "EUR", quote: "USD", rate: 1.08 }, ] - result = CarryForward.latest(rows, date: Date.new(2024, 1, 20), lookback: 14) + result = CarryForward.apply(rows, date: Date.new(2024, 1, 20), lookback: 14) _(result).must_be_empty end @@ -36,7 +36,7 @@ { date: Date.new(2024, 1, 1), provider: "ECB", base: "EUR", quote: "USD", rate: 1.08 }, ] - result = CarryForward.latest(rows, date: Date.new(2024, 1, 15), lookback: 14) + result = CarryForward.apply(rows, date: Date.new(2024, 1, 15), lookback: 14) _(result.size).must_equal(1) end @@ -46,7 +46,7 @@ { date: Date.new(2024, 1, 10), provider: "ECB", base: "EUR", quote: "USD", rate: 1.08 }, ] - result = CarryForward.latest(rows, date: Date.new(2024, 1, 9)) + result = CarryForward.apply(rows, date: Date.new(2024, 1, 9)) _(result).must_be_empty end @@ -57,109 +57,14 @@ { date: Date.new(2024, 1, 3), provider: "ECB", base: "EUR", quote: "GBP", rate: 0.86 }, ] - result = CarryForward.latest(rows, date: Date.new(2024, 1, 6)) + result = CarryForward.apply(rows, date: Date.new(2024, 1, 6)) _(result.size).must_equal(2) _(result.map { |r| r[:quote] }.sort).must_equal(["GBP", "USD"]) end it "returns empty array for empty input" do - _(CarryForward.latest([], date: Date.new(2024, 1, 6))).must_be_empty - end - end - - describe ".enrich" do - it "carries forward rates from prior days into dates within the range" do - friday = Date.new(2024, 1, 5) - saturday = Date.new(2024, 1, 6) - - rows = [ - { date: friday, provider: "ECB", base: "EUR", quote: "USD", rate: 1.08 }, - { date: friday, provider: "BOC", base: "CAD", quote: "USD", rate: 0.74 }, - { date: saturday, provider: "HNB", base: "EUR", quote: "USD", rate: 1.09 }, - ] - - result = CarryForward.enrich(rows, range: saturday..saturday) - - _(result.keys).must_equal([saturday]) - providers = result[saturday].map { |r| r[:provider] }.sort - - _(providers).must_equal(["BOC", "ECB", "HNB"]) - end - - it "preserves original dates on carried-forward rows" do - friday = Date.new(2024, 1, 5) - saturday = Date.new(2024, 1, 6) - - rows = [ - { date: friday, provider: "ECB", base: "EUR", quote: "USD", rate: 1.08 }, - { date: saturday, provider: "HNB", base: "EUR", quote: "USD", rate: 1.09 }, - ] - - result = CarryForward.enrich(rows, range: saturday..saturday) - ecb = result[saturday].find { |r| r[:provider] == "ECB" } - - _(ecb[:date]).must_equal(friday) - end - - it "excludes carry-forward beyond the lookback window" do - old = Date.new(2024, 1, 1) - target = Date.new(2024, 1, 8) - - rows = [ - { date: old, provider: "ECB", base: "EUR", quote: "USD", rate: 1.08 }, - { date: target, provider: "HNB", base: "EUR", quote: "USD", rate: 1.09 }, - ] - - result = CarryForward.enrich(rows, range: target..target, lookback: 5) - providers = result[target].map { |r| r[:provider] } - - _(providers).must_include("HNB") - _(providers).wont_include("ECB") - end - - it "only returns dates within the target range" do - friday = Date.new(2024, 1, 5) - saturday = Date.new(2024, 1, 6) - sunday = Date.new(2024, 1, 7) - - rows = [ - { date: friday, provider: "ECB", base: "EUR", quote: "USD", rate: 1.08 }, - { date: saturday, provider: "HNB", base: "EUR", quote: "USD", rate: 1.09 }, - ] - - result = CarryForward.enrich(rows, range: saturday..sunday) - - _(result.keys).must_equal([saturday]) - _(result).wont_include(friday) - end - - it "picks the most recent rate per provider within the lookback" do - wed = Date.new(2024, 1, 3) - fri = Date.new(2024, 1, 5) - sat = Date.new(2024, 1, 6) - - rows = [ - { date: wed, provider: "ECB", base: "EUR", quote: "USD", rate: 1.07 }, - { date: fri, provider: "ECB", base: "EUR", quote: "USD", rate: 1.08 }, - { date: sat, provider: "HNB", base: "EUR", quote: "USD", rate: 1.09 }, - ] - - result = CarryForward.enrich(rows, range: sat..sat) - ecb = result[sat].find { |r| r[:provider] == "ECB" } - - _(ecb[:rate]).must_equal(1.08) - _(ecb[:date]).must_equal(fri) - end - - it "returns empty hash when no dates have data in range" do - rows = [ - { date: Date.new(2024, 1, 1), provider: "ECB", base: "EUR", quote: "USD", rate: 1.08 }, - ] - - result = CarryForward.enrich(rows, range: Date.new(2024, 2, 1)..Date.new(2024, 2, 2)) - - _(result).must_be_empty + _(CarryForward.apply([], date: Date.new(2024, 1, 6))).must_be_empty end end end diff --git a/spec/rate_spec.rb b/spec/rate_spec.rb index e048a3f..62977c6 100644 --- a/spec/rate_spec.rb +++ b/spec/rate_spec.rb @@ -5,20 +5,20 @@ require "carry_forward" describe Rate do - describe CarryForward, ".latest" do + describe CarryForward, ".apply" do it "returns latest available rates on given date" do date = Fixtures.latest_date rows = Rate.where(date: (date - 14)..date).naked.all - data = CarryForward.latest(rows, date:) + data = CarryForward.apply(rows, date:) - _(data.sample[:date]).must_equal(date) + _(data.map { |r| r[:date] }.max).must_equal(date) end it "snaps to nearest prior date when requested date has no rates" do sunday = Fixtures.recent_sunday friday = Fixtures.preceding_friday(sunday) rows = Rate.where(provider: "ECB", date: (sunday - 14)..sunday).naked.all - data = CarryForward.latest(rows, date: sunday) + data = CarryForward.apply(rows, date: sunday) _(data.map { |r| r[:date] }.uniq).must_equal([friday]) end @@ -26,7 +26,7 @@ it "includes each provider's most recent date" do date = Fixtures.latest_date rows = Rate.where(date: (date - 14)..date).naked.all - data = CarryForward.latest(rows, date:) + data = CarryForward.apply(rows, date:) providers = data.map { |r| r[:provider] }.uniq.sort _(providers).must_include("ECB") @@ -38,7 +38,7 @@ Rate.dataset.insert(date: date - 20, base: "EUR", quote: "XTS", rate: 1.08, provider: "STALE") rows = Rate.where(date: (date - 14)..date).naked.all - data = CarryForward.latest(rows, date:) + data = CarryForward.apply(rows, date:) providers = data.map { |r| r[:provider] }.uniq _(providers).must_include("ECB") @@ -50,7 +50,7 @@ Rate.dataset.insert(date: date - 10, base: "USD", quote: "XTS", rate: 0.92, provider: "FRED") rows = Rate.where(date: (date - 14)..date).naked.all - data = CarryForward.latest(rows, date:) + data = CarryForward.apply(rows, date:) providers = data.map { |r| r[:provider] }.uniq _(providers).must_include("ECB") @@ -63,7 +63,7 @@ Rate.dataset.insert(date: older_date, base: "XTS", quote: "PLN", rate: 0.05, provider: "ECB") rows = Rate.where(date: (date - 14)..date).naked.all - data = CarryForward.latest(rows, date:) + data = CarryForward.apply(rows, date:) quotes = data.select { |r| r[:provider] == "ECB" }.map { |r| r[:quote] } _(quotes).must_include("PLN") @@ -73,18 +73,18 @@ date = Date.parse("1901-01-01") rows = Rate.where(date: (date - 14)..date).naked.all - _(CarryForward.latest(rows, date:)).must_be_empty + _(CarryForward.apply(rows, date:)).must_be_empty end it "returns latest rates when client date is ahead of server" do future_date = Date.today + 1 rows = Rate.where(date: (future_date - 14)..future_date).naked.all - data = CarryForward.latest(rows, date: future_date) + data = CarryForward.apply(rows, date: future_date) _(data).wont_be_empty today_rows = Rate.where(date: (Date.today - 14)..Date.today).naked.all - today_data = CarryForward.latest(today_rows, date: Date.today) + today_data = CarryForward.apply(today_rows, date: Date.today) _(data.map { |r| r[:date] }.uniq.sort).must_equal(today_data.map { |r| r[:date] }.uniq.sort) end diff --git a/spec/versions/v2/rate_query_spec.rb b/spec/versions/v2/rate_query_spec.rb index 5f6566b..fd640c6 100644 --- a/spec/versions/v2/rate_query_spec.rb +++ b/spec/versions/v2/rate_query_spec.rb @@ -62,21 +62,35 @@ module Versions _(query.to_a.first[:date]).must_equal(friday.to_s) end - it "uses target date for carried-forward quotes" do - # Fixtures have ECB/BOC/BOJ on business days only. On Saturday, carry-forward brings in Friday's rates. - # Without the target_date fix, quotes only present via carry-forward get Friday's date in the Saturday - # blend, producing duplicate (date, quote) pairs. Add a Saturday row from BOC so Saturday becomes a - # target date. - friday = Fixtures.preceding_friday(Fixtures.recent_sunday) - saturday = friday + 1 - Rate.dataset.insert(date: saturday, base: "CAD", quote: "USD", rate: 0.74, provider: "BOC") - - query = V2::RateQuery.new(from: friday.to_s, to: saturday.to_s) + it "stamps each row with its own observation date on latest path" do + # Carry-forward on the latest path should report each pair's actual observation date, + # not flatten everything to the batch max. + stale_date = Fixtures.latest_date - 5 + Rate.dataset.insert(date: stale_date, base: "EUR", quote: "RON", rate: 4.97, provider: "ECB") + + query = V2::RateQuery.new(date: Fixtures.latest_date.to_s) results = query.to_a - pairs = results.map { |r| [r[:date], r[:quote]] } + ron = results.find { |r| r[:base] == "EUR" && r[:quote] == "RON" } + usd = results.find { |r| r[:base] == "EUR" && r[:quote] == "USD" } + + _(ron[:date]).must_equal(stale_date.to_s) + _(usd[:date]).must_equal(Fixtures.latest_date.to_s) + end + + it "does not carry forward in range queries" do + # A pair published only on a single date inside the range should appear once on that date, + # not be carried forward into subsequent days. + from = Fixtures.latest_date - 6 + to = Fixtures.latest_date + stale_date = Fixtures.latest_date - 5 + Rate.dataset.insert(date: stale_date, base: "EUR", quote: "RON", rate: 4.97, provider: "ECB") + + query = V2::RateQuery.new(from: from.to_s, to: to.to_s) + ron_rows = query.to_a.select { |r| r[:base] == "EUR" && r[:quote] == "RON" } - _(pairs).must_equal(pairs.uniq) + _(ron_rows.size).must_equal(1) + _(ron_rows.first[:date]).must_equal(stale_date.to_s) end describe "with expand=providers" do