blocks-transformer/pre_processing.py

import requests
import logging

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s - %(message)s",
)
logger = logging.getLogger(__name__)

def process_record(url, record):
    try:
        response = requests.post(
            url,
            json={"record": record},
            headers={"Content-Type": "application/json"}
        )

        if response.status_code == 200:
            output = response.json()
            logger.info(f"Pre Processed record: {record['application_key']} - Response: {output}")
            return output
        elif response.status_code == 404:
            # Special-case handling for "CLUSTER_NOT_FOUND" (or any 404).
            # Return a benign structure that includes 'hd_key' and 'matches'
            # so post_processing does not fail when it does:
            #   record_id = data.get('hd_key')
            #   connected_records = data.get('matches', []) + [record_id]
            logger.warning(
                f"Ignoring 404 for record {record['application_key']}. "
                f"Status: 404, Response: {response.text}"
            )
            return {
                "application_key": record.get("application_key"),
                "hd_key": None,
                "matches": []
            }
        else:
            logger.error(f"Failed to process record {record['application_key']}. Status: {response.status_code}, Response: {response.text}")
            return {"error": response.text}
    except Exception as e:
        logger.error(f"Error processing record {record['application_key']}: {str(e)}")
        return {"error": str(e)}