Skip to content

Commit ed96d76

Browse files
Datayama38Datayama38
authored
Fix/dbt dashboard (#2895)
* wip dbt script for dashboard * fix: dbt dashboard
1 parent df4f02f commit ed96d76

8 files changed

Lines changed: 104 additions & 121 deletions

File tree

‎.vscode/settings.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,5 +52,6 @@
5252
},
5353
"typescript.preferences.importModuleSpecifier": "relative",
5454
"typescript.preferences.quoteStyle": "single",
55-
"typescript.updateImportsOnFileMove.enabled": "always"
55+
"typescript.updateImportsOnFileMove.enabled": "always",
56+
"dbt.enableNewLineagePanel": true
5657
}

‎api/src/pdc/services/dashboard/providers/JourneysRepository.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ export class JourneysRepository implements JourneysRepositoryInterface {
7575
FROM ${raw(this.tableByMonth)}
7676
WHERE ${join(filters, " AND ")}
7777
GROUP BY 1,2,3,4
78+
ORDER BY year, month
7879
`;
7980
const rows = await this.pgConnection.query<JourneysIncentiveByMonthResultInterface>(query);
8081
return rows;

‎dbt/models/dashboard/_carpools_by_day.yml‎

Lines changed: 0 additions & 20 deletions
This file was deleted.

‎dbt/models/dashboard/carpools_by_day.sql‎

Lines changed: 0 additions & 34 deletions
This file was deleted.

‎dbt/models/dashboard/operators_by_day.sql‎

Lines changed: 78 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,81 @@
1-
{{ config(
1+
{{
2+
config(
23
materialized='incremental',
34
unique_key=['territory_id', 'direction', 'start_date', 'operator_id'],
4-
post_hook=[
5-
'DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = '"'operators_by_day_pkey'"') THEN ALTER TABLE {{ this }} ADD CONSTRAINT operators_by_day_pkey PRIMARY KEY (territory_id, direction, start_date, operator_id); END IF; END $$;'
6-
'CREATE INDEX IF NOT EXISTS operators_by_day_idx ON {{ this }} using btree(territory_id, direction, start_date, operator_id)',
5+
indexes = [
6+
{
7+
'columns':[
8+
'territory_id',
9+
'direction',
10+
'start_date',
11+
'operator_id',
12+
],
13+
'unique':true
14+
}
715
]
816
)
917
}}
1018
with directions as (
11-
select
12-
"from" as territory_id,
13-
'from' as direction,
14-
start_date,
15-
operator_id,
16-
operator_name,
17-
sum(journeys) as journeys,
18-
coalesce(
19-
sum(journeys) filter (where "from" = "to"), 0
20-
) as intra_journeys,
21-
sum(incented_journeys) as incented_journeys,
22-
coalesce(
23-
sum(incented_journeys) filter (where "from" = "to"), 0
24-
) as intra_incented_journeys,
25-
sum(incentive_amount) as incentive_amount,
26-
coalesce(
27-
sum(incentive_amount) filter (where "from" = "to"), 0
28-
) as intra_incentive_amount
29-
from {{ ref('carpools_by_day') }}
30-
{% if is_incremental() %}
31-
where start_date >= (select max(start_date) from {{ this }})
32-
{% endif %}
33-
group by 1, 3, 4, 5
34-
union
35-
select
36-
"to" as territory_id,
37-
'to' as direction,
38-
start_date,
39-
operator_id,
40-
operator_name,
41-
sum(journeys) as journeys,
42-
0 as intra_journeys,
43-
sum(incented_journeys) as incented_journeys,
44-
0 as intra_incented_journeys,
45-
sum(incentive_amount) as incentive_amount,
46-
0 as intra_incentive_amount
47-
from {{ ref('carpools_by_day') }}
48-
{% if is_incremental() %}
49-
where start_date >= (select max(start_date) from {{ this }})
50-
{% endif %}
51-
group by 1, 3, 4, 5
19+
SELECT
20+
c.territory_id AS territory_id,
21+
'from' AS direction,
22+
a.start_datetime::date AS start_date,
23+
a.operator_id,
24+
a.operator_name,
25+
count(distinct a.carpool_id) filter (where b.territory_id = c.territory_id) AS journeys,
26+
count(distinct a.carpool_id) filter (where a.incentive_amount > 0 AND b.territory_id = c.territory_id) AS incented_journeys,
27+
sum(a.incentive_amount) filter (where b.territory_id = c.territory_id) as incentive_amount
28+
FROM {{ ref('view_dashboard_carpools') }} a
29+
LEFT JOIN {{ ref('view_perimeters_territories') }} b ON a.start_geo_code = b.arr
30+
LEFT JOIN {{ source('policy', 'policies') }} c ON a.policy_id = c._id
31+
WHERE
32+
c.territory_id IS NOT NULL
33+
{% if is_incremental() %}
34+
AND a.start_datetime::date >= (SELECT MAX(start_date) FROM {{ this }})::date
35+
{% endif %}
36+
GROUP BY
37+
1, 2, 3, 4, 5
38+
UNION
39+
SELECT
40+
c.territory_id AS territory_id,
41+
'to' AS direction,
42+
a.start_datetime::date AS start_date,
43+
a.operator_id,
44+
a.operator_name,
45+
count(distinct a.carpool_id) filter (where b.territory_id = c.territory_id) AS journeys,
46+
count(distinct a.carpool_id) filter (where a.incentive_amount > 0 AND b.territory_id = c.territory_id) AS incented_journeys,
47+
sum(a.incentive_amount) filter (where b.territory_id = c.territory_id) as incentive_amount
48+
FROM {{ ref('view_dashboard_carpools') }} a
49+
LEFT JOIN {{ ref('view_perimeters_territories') }} b ON a.end_geo_code = b.arr
50+
LEFT JOIN {{ source('policy', 'policies') }} c ON a.policy_id = c._id
51+
WHERE
52+
c.territory_id IS NOT NULL
53+
{% if is_incremental() %}
54+
AND a.start_datetime::date >= (SELECT MAX(start_date) FROM {{ this }})::date
55+
{% endif %}
56+
GROUP BY
57+
1, 2, 3, 4, 5
58+
UNION
59+
SELECT
60+
c.territory_id AS territory_id,
61+
'intra' AS direction,
62+
a.start_datetime::date AS start_date,
63+
a.operator_id,
64+
a.operator_name,
65+
count(distinct a.carpool_id) filter (where b.territory_id = c.territory_id) AS journeys,
66+
count(distinct a.carpool_id) filter (where a.incentive_amount > 0 AND b.territory_id = c.territory_id) AS incented_journeys,
67+
sum(a.incentive_amount) filter (where b.territory_id = c.territory_id) as incentive_amount
68+
FROM {{ ref('view_dashboard_carpools') }} a
69+
LEFT JOIN {{ ref('view_perimeters_territories') }} b ON a.start_geo_code = b.arr and a.end_geo_code = b.arr
70+
LEFT JOIN {{ source('policy', 'policies') }} c ON a.policy_id = c._id
71+
WHERE
72+
c.territory_id IS NOT NULL
73+
{% if is_incremental() %}
74+
AND a.start_datetime::date >= (SELECT MAX(start_date) FROM {{ this }})::date
75+
{% endif %}
76+
GROUP BY
77+
1, 2, 3, 4, 5
5278
)
53-
5479
select
5580
territory_id,
5681
start_date,
@@ -63,16 +88,15 @@ select
6388
from directions
6489
where territory_id is not null
6590
group by 1, 2, 3, 4, 5
66-
union
67-
select
68-
territory_id,
91+
UNION
92+
select
93+
territory_id,
6994
start_date,
70-
'both' as direction,
95+
'both' AS direction,
7196
operator_id,
7297
operator_name,
73-
sum(journeys) - sum(intra_journeys) as journeys,
74-
sum(incented_journeys) - sum(intra_incented_journeys) as incented_journeys,
75-
sum(incentive_amount) - sum(intra_incentive_amount) as incentive_amount
98+
sum(journeys) filter (where direction <> 'intra') - sum(journeys) filter (where direction = 'intra') as journeys,
99+
sum(incented_journeys) filter (where direction <> 'intra') - sum(incented_journeys) filter (where direction = 'intra') as incented_journeys,
100+
sum(incentive_amount) filter (where direction <> 'intra') - sum(incentive_amount) filter (where direction = 'intra') as incentive_amount
76101
from directions
77-
where territory_id is not null
78102
group by 1, 2, 3, 4, 5

‎dbt/models/dashboard/operators_by_month.sql‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,17 @@
11
{{ config(materialized='incremental',
22
unique_key=['year', 'month', 'territory_id', 'direction', 'operator_id'],
3-
post_hook=[
4-
"DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'operators_by_month_pkey') THEN ALTER TABLE {{ this }} ADD CONSTRAINT operators_by_month_pkey PRIMARY KEY (year, month, territory_id, direction, operator_id); END IF; END $$;"
5-
"CREATE INDEX IF NOT EXISTS operators_by_month_idx ON {{ this }} using btree(year, month, territory_id, direction, operator_id)",
6-
]
3+
indexes = [
4+
{
5+
'columns':[
6+
'year',
7+
'month',
8+
'territory_id',
9+
'direction',
10+
'operator_id',
11+
],
12+
'unique':true
13+
}
14+
]
715
) }}
816

917
SELECT

‎dbt/models/dashboard/view_dashboard_carpools.sql‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
{{ config(materialized='view') }}
22

33
SELECT
4-
a._id AS carpool_id,
4+
a.uuid AS carpool_id,
55
c.start_geo_code,
66
c.end_geo_code,
77
a.operator_id,
88
d.name AS operator_name,
9+
e.policy_id,
910
coalesce(e.amount, 0) AS incentive_amount,
1011
CASE
1112
WHEN
@@ -26,10 +27,11 @@ FROM {{ source('carpool', 'carpools') }} AS a
2627
LEFT JOIN {{ source('carpool', 'status') }} AS b ON a._id = b.carpool_id
2728
LEFT JOIN {{ source('carpool','geo') }} AS c ON a._id = c.carpool_id
2829
LEFT JOIN {{ source('operator','operators') }} AS d ON a.operator_id = d._id
29-
LEFT JOIN {{ source('policy','incentives') }} AS e ON a._id = e.carpool_id
30+
LEFT JOIN {{ source('policy','incentives') }} AS e ON a.operator_id = e.operator_id AND a.operator_journey_id = e.operator_journey_id
3031
WHERE
3132
date_part('year', a.start_datetime) >= 2020
3233
AND a.start_datetime < now() - INTERVAL '2' DAY
3334
AND b.acquisition_status = 'processed'
3435
AND b.fraud_status = 'passed'
3536
AND b.anomaly_status = 'passed'
37+
AND e.status = 'validated'

‎dbt/models/geo/view_perimeters_territories.sql‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{{ config(materialized='view') }}
22

3-
SELECT DISTINCT
3+
SELECT
44
c.arr,
55
c.l_arr,
66
a._id AS territory_id,
@@ -13,11 +13,12 @@ INNER JOIN
1313
{{ source('geo','perimeters') }} AS c
1414
ON
1515
(
16-
b.selector_value = c.arr
17-
OR b.selector_value = c.epci
18-
OR b.selector_value = c.aom
19-
OR b.selector_value = c.dep
20-
OR b.selector_value = c.reg
16+
case when b.selector_type = 'arr' then b.selector_value = c.arr
17+
when b.selector_type = 'epci' then b.selector_value = c.epci
18+
when b.selector_type = 'aom' then b.selector_value = c.aom
19+
when b.selector_type = 'dep' then b.selector_value = c.dep
20+
when b.selector_type = 'reg' then b.selector_value = c.reg
21+
end
2122
)
2223
AND c.year = 2024
2324
ORDER BY c.arr

0 commit comments

Comments
 (0)