Enhance db_lookup SQL with G2 features

README.md

@@ -1 +1 @@
-**Hello world!!!**
+# DB Lookup Block

config.json

@@ -1,6 +1,14 @@
 [
     {
-      "namespace": "",
-      "connectionId": ""
+      "namespace": "staging",
+      "connectionId": "56be80f8-7139-466f-b883-e4a695812d52"
+    },
+		{
+      "namespace": "production",
+      "connectionId": "cf2344a8-28d3-4b38-9456-ba9e5c347bf5"
+    },
+		{
+      "namespace": "production-beta",
+      "connectionId": "cf2344a8-28d3-4b38-9456-ba9e5c347bf5"
     }
   ]

main.sql

@@ -1 +1,279 @@
-Select * from dummy_table limit 10;
+WITH abc AS (
+    SELECT DISTINCT ON (app.application_key)
+        app.application_key,
+        CAST(app.application_timestamp AS TIMESTAMP) AS application_timestamp,
+        CAST(app.application_date_of_birth AS DATE) AS application_date_of_birth,
+        LOWER(app.application_email_address) AS application_email_address,
+        app.application_is_rejected,
+        NULLIF(app.application_ssn, '') AS application_ssn,
+        NULLIF(app.application_bank_account_number, '') AS application_bank_account_number,
+        loans.originated_loan_is_fraud,
+        loans.originated_loan_is_never_paid,
+        CAST(CASE WHEN LOWER(TRIM(thx.digital_id_first_seen::text)) IN ('', 'none', 'null') THEN NULL ELSE thx.digital_id_first_seen::date END AS DATE) AS digital_id_first_seen,
+        CAST(NULLIF(NULLIF(LOWER(TRIM(thx.summary_risk_score::text)), ''), 'none') AS DOUBLE PRECISION) AS summary_risk_score,
+        CAST(NULLIF(NULLIF(LOWER(TRIM(thx.cpu_clock::text)), ''), 'none') AS DOUBLE PRECISION) AS cpu_clock,
+        CAST(NULLIF(NULLIF(thx.true_ip_first_seen::text, ''), 'None') AS DATE) AS true_ip_first_seen,
+        CAST(NULLIF(NULLIF(thx.ssn_hash_first_seen::text, ''),'None') AS DATE) AS ssn_hash_first_seen,
+        thx.account_email_attributes,
+        CAST(NULLIF(NULLIF(LOWER(TRIM(thx.tps_ip_latitude::text)), ''), 'none') AS DOUBLE PRECISION) AS tps_ip_latitude,
+        CAST(NULLIF(NULLIF(LOWER(TRIM(thx.tps_ip_longitude::text)), ''), 'none') AS DOUBLE PRECISION) AS tps_ip_longitude,
+        CAST(NULLIF(NULLIF(thx.account_telephone_first_seen::text, ''),'None') AS DATE) AS account_telephone_first_seen,
+        CAST(NULLIF(NULLIF(thx.account_login_first_seen::text, ''),'None') AS DATE) AS account_login_first_seen,
+        CAST(NULLIF(NULLIF(LOWER(TRIM(ref.latitute_ref::text)), ''), 'none') AS DOUBLE PRECISION) AS latitute_ref,
+        CAST(NULLIF(NULLIF(LOWER(TRIM(ref.longitude_ref::text)), ''), 'none') AS DOUBLE PRECISION) AS longitude_ref,
+        CAST(NULLIF(NULLIF(LOWER(TRIM(scores.hd_score_m2::text)), ''), 'none') AS DOUBLE PRECISION) AS hd_score_m2
+        -- CAST(NULLIF(NULLIF(LOWER(TRIM(scores.hd_score_iso_m2::text)), ''), 'none') AS DOUBLE PRECISION) AS hd_score_iso_m2
+    FROM public.applications AS app
+    LEFT JOIN public.bridge_funding_lifecycle AS master
+        ON app.application_key = master.application_key
+    LEFT JOIN public.originated_loan AS loans
+        ON master.originated_loan_key = loans.originated_loan_key
+    LEFT JOIN public.thx AS thx
+        ON app.application_key = thx.application_key
+    LEFT JOIN public.uprova AS up
+        ON app.application_key = up.application_key
+    LEFT JOIN public.latitute_longitute_reference AS ref
+        ON 
+            /* safe ZIP join: strip non-digits, take first 5, cast to int on both sides */
+            CAST(SUBSTRING(REGEXP_REPLACE(up.zip::text, '[^0-9]', '', 'g') FOR 5) AS INTEGER)
+            =
+            CAST(SUBSTRING(REGEXP_REPLACE(ref.postal_code_ref::text, '[^0-9]', '', 'g') FOR 5) AS INTEGER)
+    LEFT JOIN public.scores AS scores
+        ON app.application_key = scores.application_key
+    WHERE app.application_key = $application_key
+       OR app.application_key = ANY(ARRAY$connected_application_keys::text[])
+    ORDER BY app.application_key, app.application_timestamp DESC
+),
+
+current_app AS (
+    SELECT
+        application_key AS current_app_key,
+        application_timestamp AS current_app_created_at,
+        $hd_score_m1 AS hd_score_m1,
+        $hd_score_iso_m2 AS hd_score_iso_m2,
+        hd_score_m2
+    FROM abc
+    WHERE application_key = $application_key
+),
+
+connected_apps AS (
+    SELECT
+        a.*,
+        a.application_key AS connected_app_key
+    FROM abc a
+    WHERE application_key = ANY(ARRAY$connected_application_keys::text[])
+),
+
+clusters_alt AS (
+    SELECT
+        a.*,
+        b.current_app_key,
+        b.current_app_created_at,
+        b.hd_score_m1,
+        b.hd_score_iso_m2,
+        CASE
+            WHEN LOWER(CAST(a.originated_loan_is_fraud AS TEXT)) != 'true'
+                AND a.originated_loan_is_never_paid IS NULL THEN NULL
+            WHEN ABS(EXTRACT(DAY FROM AGE(b.current_app_created_at, a.application_timestamp))) > 29
+                AND (LOWER(CAST(a.originated_loan_is_fraud AS TEXT)) = 'true'
+                    OR LOWER(CAST(a.originated_loan_is_never_paid AS TEXT)) = 'true')
+                THEN 1
+            ELSE 0
+        END AS target_connected_30,
+        CONCAT(a.application_ssn, '_', a.application_bank_account_number) AS user_connected_v2,
+        a.application_timestamp AS connected_date,
+        a.application_email_address AS connected_email,
+        a.application_is_rejected AS application_is_rejected_connected
+    FROM connected_apps a
+    CROSS JOIN current_app b
+),
+
+full_apps_past AS (
+    SELECT
+        current_app_key,
+        SUM(target_connected_30) AS target_connected_30_sum,
+        COUNT(DISTINCT CASE
+            WHEN LOWER(CAST(application_is_rejected_connected AS TEXT)) = 'true'
+            THEN connected_app_key
+            ELSE NULL
+        END) AS rejected_app_count
+    FROM clusters_alt
+    WHERE current_app_key != connected_app_key
+    GROUP BY current_app_key
+),
+
+clusters AS (
+    SELECT
+        a.current_app_key AS application_key,
+        CAST(MAX(hd_score_m1) AS FLOAT) AS hd_score_m1,
+        CAST(MAX(hd_score_iso_m2) AS FLOAT) as hd_score_iso_m2,
+        COUNT(DISTINCT a.user_connected_v2) AS cluster_size_users_v2,
+        MAX(b.target_connected_30_sum) AS target_connected_30_sum,
+        COUNT(DISTINCT a.connected_email) AS email_cnt,
+        MAX(b.rejected_app_count) AS rejected_app_count,
+        COUNT(DISTINCT EXTRACT(DAY FROM DATE(a.connected_date))) AS app_dt_day_cnt,
+        COUNT(*) AS cluster_size
+    FROM clusters_alt a
+    LEFT JOIN full_apps_past b
+        ON a.current_app_key = b.current_app_key
+    GROUP BY 1
+),
+
+clusters_alt_g2 AS (
+    SELECT
+        b.current_app_key,
+        a.connected_app_key,
+        a.application_ssn,
+        a.application_bank_account_number,
+        a.application_is_rejected,
+        a.hd_score_m2 AS hd_score_m2_connected,
+        CASE
+            WHEN a.application_timestamp IS NULL OR a.application_date_of_birth IS NULL THEN NULL
+            ELSE DATE_PART('year', AGE(a.application_timestamp, a.application_date_of_birth))
+        END AS applicant_age_connected,
+        CASE
+            WHEN a.application_timestamp IS NULL OR a.account_telephone_first_seen IS NULL THEN NULL
+            ELSE (a.application_timestamp::date - a.account_telephone_first_seen::date)
+        END AS account_telephone_first_seen_age_connected,
+        CASE
+            WHEN a.application_timestamp IS NULL OR a.ssn_hash_first_seen IS NULL THEN NULL
+            ELSE (a.application_timestamp::date - a.ssn_hash_first_seen::date)
+        END AS ssn_hash_first_seen_age_connected,
+        CASE
+            WHEN a.application_timestamp IS NULL OR a.account_login_first_seen IS NULL THEN NULL
+            ELSE (a.application_timestamp::date - a.account_login_first_seen::date)
+        END AS account_login_first_seen_age_connected,
+        CASE
+            WHEN a.application_timestamp IS NULL OR a.digital_id_first_seen IS NULL THEN NULL
+            ELSE (a.application_timestamp::date - a.digital_id_first_seen::date)
+        END AS digital_id_first_seen_age_connected,
+        CASE
+            WHEN a.application_timestamp IS NULL OR a.true_ip_first_seen IS NULL THEN NULL
+            ELSE (a.application_timestamp::date - a.true_ip_first_seen::date)
+        END AS true_ip_first_seen_age_connected,
+        CASE
+            WHEN a.latitute_ref IS NULL OR a.longitude_ref IS NULL
+                OR a.tps_ip_latitude IS NULL OR a.tps_ip_longitude IS NULL
+                THEN NULL
+            ELSE 2 * 6371.0088 * ASIN(
+                LEAST(
+                    1.0,
+                    SQRT(
+                        POWER(SIN(((a.tps_ip_latitude - a.latitute_ref) * 0.017453292519943295) / 2), 2)
+                        + COS(a.latitute_ref * 0.017453292519943295) * COS(a.tps_ip_latitude * 0.017453292519943295)
+                        * POWER(SIN(((a.tps_ip_longitude - a.longitude_ref) * 0.017453292519943295) / 2), 2)
+                    )
+                )
+            )
+        END AS dist_em_ip_ref_km_connected,
+        CASE
+            WHEN LOWER(a.account_email_attributes) ILIKE '%challenged%' THEN 1
+            ELSE 0
+        END AS account_email_attributes_challenged_connected,
+        a.summary_risk_score AS summary_risk_score_connected,
+        a.cpu_clock AS cpu_clock_connected,
+        CONCAT(a.application_ssn, '_', a.application_bank_account_number) AS connected_user_id
+    FROM connected_apps a
+    CROSS JOIN current_app b
+),
+
+full_apps_past_g2 AS (
+    SELECT
+        current_app_key,
+        COUNT(DISTINCT connected_user_id) AS connected_user_count,
+        COUNT(DISTINCT CASE
+            WHEN LOWER(CAST(application_is_rejected AS TEXT)) = 'true'
+            THEN connected_user_id
+        END) AS rejected_app_count_g2,
+        MAX(hd_score_m2_connected) AS hd_score_m2_connected_max,
+        AVG(hd_score_m2_connected) AS hd_score_m2_connected_avg,
+        MAX(applicant_age_connected) AS applicant_age_connected_max,
+        AVG(applicant_age_connected) AS applicant_age_connected_avg,
+        MIN(account_telephone_first_seen_age_connected) AS account_tel_first_seen_min_conn,
+        MAX(account_telephone_first_seen_age_connected) AS account_tel_first_seen_max_conn,
+        AVG(account_telephone_first_seen_age_connected) AS account_tel_first_seen_avg_conn,
+        MIN(cpu_clock_connected) AS cpu_clock_min_conn,
+        MAX(cpu_clock_connected) AS cpu_clock_max_conn,
+        MIN(ssn_hash_first_seen_age_connected) AS ssn_hash_first_seen_min_conn,
+        AVG(ssn_hash_first_seen_age_connected) AS ssn_hash_first_seen_avg_conn,
+        MIN(account_login_first_seen_age_connected) AS account_login_first_seen_min_conn,
+        MAX(account_login_first_seen_age_connected) AS account_login_first_seen_max_conn,
+        STDDEV_SAMP(account_login_first_seen_age_connected) AS account_login_first_seen_stddev_conn,
+        MAX(digital_id_first_seen_age_connected) AS digital_id_first_seen_max_conn,
+        MIN(true_ip_first_seen_age_connected) AS true_ip_first_seen_min_conn,
+        MAX(true_ip_first_seen_age_connected) AS true_ip_first_seen_max_conn,
+        MIN(dist_em_ip_ref_km_connected) AS dist_em_ip_ref_km_min_conn,
+        MAX(summary_risk_score_connected) AS summary_risk_score_max_conn,
+        COUNT(DISTINCT CASE
+            WHEN account_email_attributes_challenged_connected = 1 THEN connected_user_id
+        END) AS acc_email_attr_challenged_1_conn
+    FROM clusters_alt_g2
+    WHERE current_app_key != connected_app_key
+    GROUP BY current_app_key
+),
+
+clusters_g2 AS (
+    SELECT
+        cur.current_app_key AS application_key,
+        cur.hd_score_m2 AS hd_score_m2,
+        agg.hd_score_m2_connected_avg,
+        agg.hd_score_m2_connected_max,
+        agg.applicant_age_connected_max,
+        agg.applicant_age_connected_avg,
+        agg.account_tel_first_seen_min_conn,
+        agg.account_tel_first_seen_max_conn,
+        agg.account_tel_first_seen_avg_conn,
+        agg.ssn_hash_first_seen_min_conn,
+        agg.ssn_hash_first_seen_avg_conn,
+        agg.account_login_first_seen_min_conn,
+        agg.account_login_first_seen_stddev_conn,
+        (agg.account_login_first_seen_max_conn - agg.account_login_first_seen_min_conn) AS account_login_first_seen_range_conn,
+        agg.digital_id_first_seen_max_conn,
+        agg.true_ip_first_seen_min_conn,
+        agg.true_ip_first_seen_max_conn,
+        (agg.cpu_clock_max_conn - agg.cpu_clock_min_conn) AS cpu_clock_range_conn,
+        agg.summary_risk_score_max_conn,
+        CASE
+            WHEN agg.connected_user_count IS NULL OR agg.connected_user_count = 0 THEN NULL
+            ELSE (agg.acc_email_attr_challenged_1_conn::DOUBLE PRECISION / agg.connected_user_count::DOUBLE PRECISION) * 100
+        END AS pct_acc_email_attr_challenged_1_conn,
+        agg.rejected_app_count_g2,
+        agg.dist_em_ip_ref_km_min_conn
+    FROM current_app cur
+    LEFT JOIN full_apps_past_g2 agg
+        ON cur.current_app_key = agg.current_app_key
+)
+
+SELECT
+    COALESCE(MAX(c.hd_score_m1), $hd_score_m1) AS hd_score_m1,
+    COALESCE(MAX(c.hd_score_iso_m2), $hd_score_iso_m2) AS hd_score_iso_m2,
+    COALESCE(MAX(c.cluster_size_users_v2), 0) AS cluster_size_users_v2,
+    COALESCE(MAX(c.target_connected_30_sum), 0) AS target_connected_30_sum,
+    COALESCE(MAX(c.email_cnt), 0) AS email_cnt,
+    COALESCE(MAX(c.rejected_app_count), 0) AS rejected_app_count,
+    COALESCE(MAX(c.app_dt_day_cnt), 0) AS app_dt_day_cnt,
+    COALESCE(MAX(c.cluster_size), 0) AS cluster_size,
+    MAX(g2.hd_score_m2) AS hd_score_m2,
+    MAX(g2.hd_score_m2_connected_avg) AS hd_score_m2_connected_avg,
+    MAX(g2.rejected_app_count_g2) AS rejected_app_count_g2,
+    MAX(g2.summary_risk_score_max_conn) AS summary_risk_score_max_conn,
+    MAX(g2.cpu_clock_range_conn) AS cpu_clock_range_conn,
+    MAX(g2.ssn_hash_first_seen_avg_conn) AS ssn_hash_first_seen_avg_conn,
+    MAX(g2.account_tel_first_seen_avg_conn) AS account_tel_first_seen_avg_conn,
+    MAX(g2.account_tel_first_seen_min_conn) AS account_tel_first_seen_min_conn,
+    MAX(g2.account_login_first_seen_stddev_conn) AS account_login_first_seen_stddev_conn,
+    MAX(g2.account_login_first_seen_min_conn) AS account_login_first_seen_min_conn,
+    MAX(g2.ssn_hash_first_seen_min_conn) AS ssn_hash_first_seen_min_conn,
+    MAX(g2.account_tel_first_seen_max_conn) AS account_tel_first_seen_max_conn,
+    MAX(g2.applicant_age_connected_max) AS applicant_age_connected_max,
+    MAX(g2.applicant_age_connected_avg) AS applicant_age_connected_avg,
+    MAX(g2.digital_id_first_seen_max_conn) AS digital_id_first_seen_max_conn,
+    MAX(g2.account_login_first_seen_range_conn) AS account_login_first_seen_range_conn,
+    MAX(g2.true_ip_first_seen_min_conn) AS true_ip_first_seen_min_conn,
+    MAX(g2.true_ip_first_seen_max_conn) AS true_ip_first_seen_max_conn,
+    MAX(g2.hd_score_m2_connected_max) AS hd_score_m2_connected_max,
+    MAX(g2.pct_acc_email_attr_challenged_1_conn) AS pct_acc_email_attr_challenged_1_conn,
+    MAX(g2.dist_em_ip_ref_km_min_conn) AS dist_em_ip_ref_km_min_conn
+FROM clusters c
+LEFT JOIN clusters_g2 g2
+    ON c.application_key = g2.application_key;

request_schema.json

@@ -1 +1,26 @@
-{}
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "application_key": {
+      "type": ["string", "null"],
+      "description": "Unique identifier for the application."
+    },
+    "hd_score_m1": {
+      "type": ["number", "null"],
+      "description": "HD fraud score M1."
+    },
+    "hd_score_iso_m2": {
+      "type": ["number", "null"],
+      "description": "HD fraud score M2."
+    },
+    "connected_application_keys": {
+      "type": "array",
+      "items": {
+        "type": ["string", "null"]
+      },
+      "description": "List of connected application keys associated with this application."
+    }
+  },
+  "required": []
+}

response_schema.json

@@ -1 +1,11 @@
-{}
+{
+  "$schema": "http://json-schema.org/draft-07/schema",
+  "type": "object",
+  "properties": {
+    "results": {
+      "type": ["array", "null"],
+      "items": {"type": "object"}
+    }
+  },
+  "required": []
+}