Skip to content

Commit e6e1eb0

Browse files
smnorrissmnorris
committed
wrap all sql files in begin, commit, note this is v0.7.2
1 parent a638de5 commit e6e1eb0

6 files changed

Lines changed: 154 additions & 144 deletions

File tree

‎CHANGES.txt‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,12 @@ Changes
33

44
All issue numbers are relative to https://github.com/smnorris/fwapg/issues
55

6-
0.7.2 (2026-04- )
6+
0.7.2 (2026-04-17)
77
------------------
88
- pull source data from nrs.objectstore
99
- consolidate all schema definition scripts into schema.sql
1010
- load db schema to docker container on build, enabling pull of functional (empty) database
11-
- add migrations folder and script for future changes
12-
- enable per watershed group data loads
11+
- add migrations scripts and docs to standardize how future changes are applied
1312

1413
0.7.1 (2025-11-07)
1514
------------------

‎docs/conf.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
author = 'Simon Norris'
2323

2424
# The full version, including alpha/beta/rc tags
25-
release = '0.7.2dev0'
25+
release = '0.7.2'
2626

2727

2828
# -- General configuration ---------------------------------------------------

‎extras/README.md‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
# Extras
22

33
Value added FWA data that generally requires several hours to generate.
4+
Rather than re-generate the lookups each time the database is loaded, `fwapg` loads this data from files cached to nrs.objectstore.
45

5-
Rather than re-generate the lookups each time the database is loaded, `fwapg` defaults to loading the tables from cached csvs.
6+
## Updates
67

7-
If updates to the lookups are required, regenerate the csv files using these scripts.
8-
9-
10-
# Publication
11-
12-
To upload data to BC object storage on completion of scripts:
8+
Updates to the lookups are generally required whenever the source FWA geometries and codes have changed.
9+
Update source FWA files in your database, then regenerate the cached data files using scripts in this folder.
10+
When complete, upload to nrs.objectstore:
1311

1412
aws s3 cp channel_width/fwa_stream_networks_channel_width.csv.gz s3://bchamp/fwapg/fwa_stream_networks_channel_width.csv.gz --acl public-read
1513
aws s3 cp discharge/fwa_stream_networks_discharge.csv.gz s3://bchamp/fwapg/fwa_stream_networks_discharge.csv.gz --acl public-read
1614
aws s3 cp precipitation/fwa_stream_networks_mean_annual_precip.csv.gz s3://bchamp/fwapg/fwa_stream_networks_mean_annual_precip.csv.gz --acl public-read
1715
aws s3 cp lookups/fwa_assessment_watersheds_lut.csv.gz s3://bchamp/fwapg/fwa_assessment_watersheds_lut.csv.gz --acl public-read
1816
aws s3 cp lookups/fwa_assessment_watersheds_streams_lut.csv.gz s3://bchamp/fwapg/fwa_assessment_watersheds_streams_lut.csv.gz --acl public-read
1917
aws s3 cp lookups/fwa_waterbodies_upstream_area.csv.gz s3://bchamp/fwapg/fwa_waterbodies_upstream_area.csv.gz --acl public-read
20-
aws s3 cp lookups/fwa_watersheds_upstream_area.csv.gz s3://bchamp/fwapg/fwa_watersheds_upstream_area.csv.gz --acl public-read
18+
aws s3 cp lookups/fwa_watersheds_upstream_area.csv.gz s3://bchamp/fwapg/fwa_watersheds_upstream_area.csv.gz --acl public-read
19+
20+
Note that no metadata is currently provided to link versions of these add-ons to a given version of the FWA data (this is a to-do).

‎load/fwa_stream_networks_order_parent.sql‎

Lines changed: 34 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,46 @@
11
-- create lookup linking streams to their parent order
22
-- (generally used for finding small channels that drain into major rivers)
3+
BEGIN;
34

4-
-- load primary channels
5-
insert into whse_basemapping.fwa_stream_networks_order_parent
6-
(blue_line_key, stream_order_parent)
5+
-- load primary channels
6+
insert into whse_basemapping.fwa_stream_networks_order_parent
7+
(blue_line_key, stream_order_parent)
8+
select distinct on (a.blue_line_key)
9+
a.blue_line_key,
10+
b.stream_order as stream_order_parent
11+
from whse_basemapping.fwa_stream_networks_sp a
12+
left outer join whse_basemapping.fwa_stream_networks_sp b
13+
on a.wscode_ltree = b.localcode_ltree
14+
where
15+
a.watershed_group_code = :'wsg'
16+
and a.blue_line_key = a.watershed_key
17+
and b.blue_line_key = b.watershed_key
18+
and a.wscode_ltree <@ '999' is false
19+
and a.edge_type != 6010
20+
and a.localcode_ltree is not null
21+
and a.blue_line_key != b.blue_line_key
22+
order by a.blue_line_key, b.stream_order desc
23+
on conflict do nothing;
24+
25+
-- load side channels
26+
-- (use the order of non-side channel stream with equivalent watershed codes
27+
-- as the side channel)
28+
-- NOTE - this will not populate parent order for side channels that do not
29+
-- have local codes
30+
insert into whse_basemapping.fwa_stream_networks_order_parent (
31+
blue_line_key,
32+
stream_order_parent
33+
)
734
select distinct on (a.blue_line_key)
835
a.blue_line_key,
936
b.stream_order as stream_order_parent
1037
from whse_basemapping.fwa_stream_networks_sp a
1138
left outer join whse_basemapping.fwa_stream_networks_sp b
12-
on a.wscode_ltree = b.localcode_ltree
39+
on (a.wscode_ltree = b.wscode_ltree and
40+
a.localcode_ltree = b.localcode_ltree)
1341
where
1442
a.watershed_group_code = :'wsg'
15-
and a.blue_line_key = a.watershed_key
43+
and a.blue_line_key != a.watershed_key
1644
and b.blue_line_key = b.watershed_key
1745
and a.wscode_ltree <@ '999' is false
1846
and a.edge_type != 6010
@@ -21,29 +49,4 @@ insert into whse_basemapping.fwa_stream_networks_order_parent
2149
order by a.blue_line_key, b.stream_order desc
2250
on conflict do nothing;
2351

24-
-- load side channels
25-
-- (use the order of non-side channel stream with equivalent watershed codes
26-
-- as the side channel)
27-
-- NOTE - this will not populate parent order for side channels that do not
28-
-- have local codes
29-
insert into whse_basemapping.fwa_stream_networks_order_parent (
30-
blue_line_key,
31-
stream_order_parent
32-
)
33-
select distinct on (a.blue_line_key)
34-
a.blue_line_key,
35-
b.stream_order as stream_order_parent
36-
from whse_basemapping.fwa_stream_networks_sp a
37-
left outer join whse_basemapping.fwa_stream_networks_sp b
38-
on (a.wscode_ltree = b.wscode_ltree and
39-
a.localcode_ltree = b.localcode_ltree)
40-
where
41-
a.watershed_group_code = :'wsg'
42-
and a.blue_line_key != a.watershed_key
43-
and b.blue_line_key = b.watershed_key
44-
and a.wscode_ltree <@ '999' is false
45-
and a.edge_type != 6010
46-
and a.localcode_ltree is not null
47-
and a.blue_line_key != b.blue_line_key
48-
order by a.blue_line_key, b.stream_order desc
49-
on conflict do nothing;
52+
COMMIT;

‎load/fwa_streams.sql‎

Lines changed: 65 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -1,61 +1,65 @@
1-
insert into whse_basemapping.fwa_streams (
2-
linear_feature_id,
3-
edge_type,
4-
blue_line_key,
5-
watershed_key,
6-
wscode,
7-
localcode,
8-
watershed_group_code,
9-
downstream_route_measure,
10-
upstream_route_measure,
11-
length_metre,
12-
waterbody_key,
13-
gnis_name,
14-
stream_order,
15-
stream_magnitude,
16-
feature_code,
17-
gradient,
18-
left_right_tributary,
19-
stream_order_parent,
20-
stream_order_max,
21-
upstream_area_ha,
22-
map_upstream,
23-
channel_width,
24-
channel_width_source,
25-
mad_m3s,
26-
geom
27-
)
28-
select
29-
s.linear_feature_id,
30-
s.edge_type,
31-
s.blue_line_key,
32-
s.watershed_key,
33-
s.wscode_ltree as wscode,
34-
s.localcode_ltree as localcode,
35-
s.watershed_group_code,
36-
s.downstream_route_measure,
37-
s.upstream_route_measure,
38-
s.length_metre,
39-
s.waterbody_key,
40-
s.gnis_name,
41-
s.stream_order,
42-
s.stream_magnitude,
43-
s.feature_code,
44-
s.gradient,
45-
s.left_right_tributary,
46-
op.stream_order_parent,
47-
om.stream_order_max,
48-
ua.upstream_area_ha,
49-
p.map_upstream,
50-
cw.channel_width,
51-
cw.channel_width_source,
52-
mad.mad_m3s,
53-
s.geom
54-
from whse_basemapping.fwa_stream_networks_sp s
55-
left outer join whse_basemapping.fwa_streams_watersheds_lut l on s.linear_feature_id = l.linear_feature_id
56-
inner join whse_basemapping.fwa_watersheds_upstream_area ua on l.watershed_feature_id = ua.watershed_feature_id
57-
left outer join whse_basemapping.fwa_stream_networks_order_parent op on s.blue_line_key = op.blue_line_key
58-
left outer join whse_basemapping.fwa_stream_networks_order_max om on s.blue_line_key = om.blue_line_key
59-
left outer join whse_basemapping.fwa_stream_networks_mean_annual_precip p on s.wscode_ltree = p.wscode_ltree and s.localcode_ltree = p.localcode_ltree
60-
left outer join whse_basemapping.fwa_stream_networks_channel_width cw on s.linear_feature_id = cw.linear_feature_id
61-
left outer join whse_basemapping.fwa_stream_networks_discharge mad on s.linear_feature_id = mad.linear_feature_id;
1+
BEGIN;
2+
3+
insert into whse_basemapping.fwa_streams (
4+
linear_feature_id,
5+
edge_type,
6+
blue_line_key,
7+
watershed_key,
8+
wscode,
9+
localcode,
10+
watershed_group_code,
11+
downstream_route_measure,
12+
upstream_route_measure,
13+
length_metre,
14+
waterbody_key,
15+
gnis_name,
16+
stream_order,
17+
stream_magnitude,
18+
feature_code,
19+
gradient,
20+
left_right_tributary,
21+
stream_order_parent,
22+
stream_order_max,
23+
upstream_area_ha,
24+
map_upstream,
25+
channel_width,
26+
channel_width_source,
27+
mad_m3s,
28+
geom
29+
)
30+
select
31+
s.linear_feature_id,
32+
s.edge_type,
33+
s.blue_line_key,
34+
s.watershed_key,
35+
s.wscode_ltree as wscode,
36+
s.localcode_ltree as localcode,
37+
s.watershed_group_code,
38+
s.downstream_route_measure,
39+
s.upstream_route_measure,
40+
s.length_metre,
41+
s.waterbody_key,
42+
s.gnis_name,
43+
s.stream_order,
44+
s.stream_magnitude,
45+
s.feature_code,
46+
s.gradient,
47+
s.left_right_tributary,
48+
op.stream_order_parent,
49+
om.stream_order_max,
50+
ua.upstream_area_ha,
51+
p.map_upstream,
52+
cw.channel_width,
53+
cw.channel_width_source,
54+
mad.mad_m3s,
55+
s.geom
56+
from whse_basemapping.fwa_stream_networks_sp s
57+
left outer join whse_basemapping.fwa_streams_watersheds_lut l on s.linear_feature_id = l.linear_feature_id
58+
inner join whse_basemapping.fwa_watersheds_upstream_area ua on l.watershed_feature_id = ua.watershed_feature_id
59+
left outer join whse_basemapping.fwa_stream_networks_order_parent op on s.blue_line_key = op.blue_line_key
60+
left outer join whse_basemapping.fwa_stream_networks_order_max om on s.blue_line_key = om.blue_line_key
61+
left outer join whse_basemapping.fwa_stream_networks_mean_annual_precip p on s.wscode_ltree = p.wscode_ltree and s.localcode_ltree = p.localcode_ltree
62+
left outer join whse_basemapping.fwa_stream_networks_channel_width cw on s.linear_feature_id = cw.linear_feature_id
63+
left outer join whse_basemapping.fwa_stream_networks_discharge mad on s.linear_feature_id = mad.linear_feature_id;
64+
65+
COMMIT;

‎load/fwa_streams_watersheds_lut.sql‎

Lines changed: 44 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -5,45 +5,49 @@
55

66
-- first, find matches based on watershed code
77
-- where there is more than one match, match to closest
8-
INSERT INTO whse_basemapping.fwa_streams_watersheds_lut
9-
(
10-
linear_feature_id,
11-
watershed_feature_id
12-
)
13-
SELECT DISTINCT ON (linear_feature_id)
14-
s.linear_feature_id,
15-
w.watershed_feature_id
16-
FROM whse_basemapping.fwa_stream_networks_sp s
17-
INNER JOIN whse_basemapping.fwa_watersheds_poly w
18-
ON (s.wscode_ltree = w.wscode_ltree AND
19-
s.localcode_ltree = w.localcode_ltree AND
20-
s.watershed_group_code = w.watershed_group_code)
21-
WHERE s.watershed_group_code = :'wsg'
22-
AND s.fwa_watershed_code NOT LIKE '999%'
23-
AND s.local_watershed_code IS NOT NULL
24-
AND s.edge_type != 6100
25-
ORDER BY s.linear_feature_id, ST_Distance(ST_LineInterpolatePoint(s.geom, .5), ST_Centroid(w.geom));
8+
BEGIN;
269

10+
INSERT INTO whse_basemapping.fwa_streams_watersheds_lut
11+
(
12+
linear_feature_id,
13+
watershed_feature_id
14+
)
15+
SELECT DISTINCT ON (linear_feature_id)
16+
s.linear_feature_id,
17+
w.watershed_feature_id
18+
FROM whse_basemapping.fwa_stream_networks_sp s
19+
INNER JOIN whse_basemapping.fwa_watersheds_poly w
20+
ON (s.wscode_ltree = w.wscode_ltree AND
21+
s.localcode_ltree = w.localcode_ltree AND
22+
s.watershed_group_code = w.watershed_group_code)
23+
WHERE s.watershed_group_code = :'wsg'
24+
AND s.fwa_watershed_code NOT LIKE '999%'
25+
AND s.local_watershed_code IS NOT NULL
26+
AND s.edge_type != 6100
27+
ORDER BY s.linear_feature_id, ST_Distance(ST_LineInterpolatePoint(s.geom, .5), ST_Centroid(w.geom));
2728

28-
-- For streams with no matching watershed based on watershed codes,
29-
-- do a spatial join, selecting watershed that intersects the midpoint of the stream,
30-
-- and where more than one watershed intersects, select the one with the closest centroid
31-
INSERT INTO whse_basemapping.fwa_streams_watersheds_lut
32-
(
33-
linear_feature_id,
34-
watershed_feature_id
35-
)
36-
SELECT DISTINCT ON (linear_feature_id)
37-
s.linear_feature_id,
38-
w.watershed_feature_id
39-
FROM whse_basemapping.fwa_stream_networks_sp s
40-
LEFT JOIN whse_basemapping.fwa_streams_watersheds_lut l
41-
ON s.linear_feature_id = l.linear_feature_id
42-
INNER JOIN whse_basemapping.fwa_watersheds_poly w
43-
ON ST_Intersects(ST_LineInterpolatePoint(s.geom, .5), w.geom)
44-
WHERE l.watershed_feature_id IS NULL -- extract only streams that are not already matched
45-
AND s.watershed_group_code = :'wsg'
46-
AND s.fwa_watershed_code NOT LIKE '999%'
47-
AND s.local_watershed_code IS NOT NULL
48-
AND s.edge_type != 6100
49-
ORDER BY s.linear_feature_id, ST_Distance(ST_LineInterpolatePoint(s.geom, .5), ST_Centroid(w.geom));
29+
30+
-- For streams with no matching watershed based on watershed codes,
31+
-- do a spatial join, selecting watershed that intersects the midpoint of the stream,
32+
-- and where more than one watershed intersects, select the one with the closest centroid
33+
INSERT INTO whse_basemapping.fwa_streams_watersheds_lut
34+
(
35+
linear_feature_id,
36+
watershed_feature_id
37+
)
38+
SELECT DISTINCT ON (linear_feature_id)
39+
s.linear_feature_id,
40+
w.watershed_feature_id
41+
FROM whse_basemapping.fwa_stream_networks_sp s
42+
LEFT JOIN whse_basemapping.fwa_streams_watersheds_lut l
43+
ON s.linear_feature_id = l.linear_feature_id
44+
INNER JOIN whse_basemapping.fwa_watersheds_poly w
45+
ON ST_Intersects(ST_LineInterpolatePoint(s.geom, .5), w.geom)
46+
WHERE l.watershed_feature_id IS NULL -- extract only streams that are not already matched
47+
AND s.watershed_group_code = :'wsg'
48+
AND s.fwa_watershed_code NOT LIKE '999%'
49+
AND s.local_watershed_code IS NOT NULL
50+
AND s.edge_type != 6100
51+
ORDER BY s.linear_feature_id, ST_Distance(ST_LineInterpolatePoint(s.geom, .5), ST_Centroid(w.geom));
52+
53+
COMMIT;

0 commit comments

Comments
 (0)