usaspending-api/development.Dockerfile at master · dorianhadley/usaspending-api · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
# Dockerfile to support local developement and tests
FROM public.ecr.aws/emr-serverless/spark/emr-7.12.0:latest

USER root

# Copy UV from uv image
COPY --from=ghcr.io/astral-sh/uv:0.7.19 /uv /uvx /bin/

# Build ARGs
ARG PROJECT_LOG_DIR=/logs
ARG PYTHON_VERSION=3.10.12
ARG SPARK_LOGGING_LEVEL=INFO

# Install dependencies
RUN dnf update \
    && dnf install -y \
        git \
        libffi-devel \
        libpq-devel \
        nodejs \
        npm \
        postgresql16 \
        sqlite-devel \
        wget \
        zlib-devel \
    && dnf clean all

# Download the Postgres JAR
RUN wget -P /usr/lib/spark/jars/ https://jdbc.postgresql.org/download/postgresql-42.7.5.jar

# Install Dredd
RUN npm install --global dredd@13.1.2

WORKDIR /usaspending-api

# Allow git operations to be run inside of the container
RUN git config --global --add safe.directory /usaspending-api

##### The following ENV vars are optimizations from https://github.com/astral-sh/uv-docker-example/blob/main/Dockerfile
##### and https://docs.astral.sh/uv/guides/integration/docker/#optimizations
# Specify .venv location outside of the project to avoid conflict
ENV UV_PROJECT_ENVIRONMENT=/usr/local/.venv

# Enable bytecode compilation
ENV UV_COMPILE_BYTECODE=1

# Copy from the cache instead of linking since it's a mounted volume
ENV UV_LINK_MODE=copy

# Install Python and dev dependencies
RUN --mount=type=cache,target=/root/.cache/uv \
    --mount=type=bind,source=.,target=/usaspending-api \
    uv sync --extra dev --extra spark --locked --python ${PYTHON_VERSION}

# Make sure uv environment is active
ENV PATH="/usr/local/.venv/bin:$PATH"

# Set Python 3.10.12 as the default Python for PySpark
ENV PYSPARK_PYTHON=/usr/local/.venv/bin/python3
ENV PYSPARK_DRIVER_PYTHON=/usr/local/.venv/bin/python3

# Configure console logging for Spark
RUN cat <<EOF | sed 's/^[[:space:]]*//' >> $SPARK_HOME/conf/log4j2.properties
	appender.console.type = Console
    appender.console.name = console
    appender.console.layout.type = PatternLayout
    appender.console.layout.pattern = [%d{yyyy-MM-dd HH:mm:ss.SSS}][%p] - %m%n

    rootLogger.level = ${SPARK_LOGGING_LEVEL}
    rootLogger.appenderRefs = stdout
    rootLogger.appenderRef.stdout.ref = console
EOF

# Update logging for Spark
RUN sed -i "s|^spark.eventLog.dir.*$|spark.eventLog.dir               file:///usaspending-api/$PROJECT_LOG_DIR/spark-events|g" $SPARK_HOME/conf/spark-defaults.conf \
    && sed -i "/spark.eventLog.enabled/s/^# //g" $SPARK_HOME/conf/spark-defaults.conf \
    && sed -i "s|spark.history.fs.logDirectory.*$|spark.history.fs.logDirectory    file:///usaspending-api/$PROJECT_LOG_DIR/spark-events|" $SPARK_HOME/conf/spark-defaults.conf

# Disable authentication for local development
RUN sed -i "s|spark.authenticate \{1,\}true|spark.authenticate false|" $SPARK_HOME/conf/spark-defaults.conf

# Ensure Python STDOUT gets sent to container logs
ENV PYTHONUNBUFFERED=1