Extended by Langfuse but minio_ctr dies

bin/create_pod_langflow.sh

@@ -1,33 +1,105 @@
 #!/bin/bash
 
-# To be run by user ncd to create pod and Langflow + Postgres containers
+# To be run by user ncd to create pod for Langflow, Langfuse and their tools
 
 set -e
 
 # Environment variables
 POD_NAME='langflow_pod'
 LANGFLOW_CTR_NAME='langflow_ctr'
-# Use a valid Langflow image tag
+POSTGRES_CTR_NAME='postgres_ctr'
+LANGFUSE_WEB_CTR_NAME='langfuse-web_ctr'
+LANGFUSE_WORKER_CTR_NAME='langfuse-worker_ctr'
+CLICKHOUSE_CTR_NAME='clickhouse_ctr'
+MINIO_CTR_NAME='minio_ctr'
+REDIS_CTR_NAME='redis_ctr'
+
 LANGFLOW_IMAGE='docker.io/langflowai/langflow:1.6.9'
-POSTGRES_CTR_NAME='postgres'
-# Match original docker-compose version
 POSTGRES_IMAGE='docker.io/library/postgres:16'
+LANGFUSE_WEB_IMAGE='docker.io/langfuse/langfuse:3'
+LANGFUSE_WORKER_IMAGE='docker.io/langfuse/langfuse-worker:3'
+CLICKHOUSE_IMAGE='docker.io/clickhouse/clickhouse-server:25.11'
+# Chainguard provides only :latest? https://images.chainguard.dev/directory/image/minio/versions
+MINIO_IMAGE='cgr.dev/chainguard/minio'
+REDIS_IMAGE='docker.io/redis:7'
+
 HOST_LOCAL_IP='127.0.0.1'
-# Expose Langflow on 8090 -> 7860; Postgres is only reachable inside the pod
+# Expose Langflow on 8090 -> 7860
 LANGFLOW_HOST_PORT='8090'
 LANGFLOW_CONTAINER_PORT='7860'
+# Expose Langfuse Web on 8091 -> 3000
+LANGFUSE_HOST_PORT='8091'
+LANGFUSE_CONTAINER_PORT='3000'
+# All other services are only reachable inside the pod
+
 BIND_DIR="$HOME/.local/share/$POD_NAME"
 LANGFLOW_DATA_DIR="$BIND_DIR/langflow-data"
 POSTGRES_DATA_DIR="$BIND_DIR/postgres-data"
+LANGFUSE_CLICKHOUSE_DATA_DIR="$BIND_DIR/langfuse-clickhouse-data"
+LANGFUSE_CLICKHOUSE_LOGS_DIR="$BIND_DIR/langfuse-clickhouse-logs"
+LANGFUSE_MINIO_DATA_DIR="$BIND_DIR/langfuse-minio-data"
 USER_SYSTEMD_DIR="$HOME/.config/systemd/user"
 
+LANGFUSE_PUBLIC_URL="http://$HOST_LOCAL_IP:$LANGFUSE_HOST_PORT"
+LANGFUSE_COMMON_ENV=(
+  -e NEXTAUTH_URL="$LANGFUSE_PUBLIC_URL"
+  -e DATABASE_URL="postgresql://langflow:langflow@localhost:5432/langfuse"
+  -e SALT="qwexoiuzxequtbfvyqewregvg"
+  -e ENCRYPTION_KEY="57db4291f0243a9f6bbbf82d3fbca1a72c58fd42f852a4d74e3db8b55bee720a" # TODO: change to a secure 64-char hex key
+  -e TELEMETRY_ENABLED=true
+  -e LANGFUSE_ENABLE_EXPERIMENTAL_FEATURES=true
+  -e CLICKHOUSE_MIGRATION_URL="clickhouse://localhost:9000"
+  -e CLICKHOUSE_URL="http://localhost:8123"
+  -e CLICKHOUSE_USER="clickhouse"
+  -e CLICKHOUSE_PASSWORD="clickhouse"
+  -e CLICKHOUSE_CLUSTER_ENABLED=false
+  -e LANGFUSE_USE_AZURE_BLOB=false
+  -e LANGFUSE_S3_EVENT_UPLOAD_BUCKET="langfuse"
+  -e LANGFUSE_S3_EVENT_UPLOAD_REGION="auto"
+  -e LANGFUSE_S3_EVENT_UPLOAD_ACCESS_KEY_ID="minio"
+  -e LANGFUSE_S3_EVENT_UPLOAD_SECRET_ACCESS_KEY="miniosecret"
+  -e LANGFUSE_S3_EVENT_UPLOAD_ENDPOINT="http://localhost:9000"
+  -e LANGFUSE_S3_EVENT_UPLOAD_FORCE_PATH_STYLE=true
+  -e LANGFUSE_S3_EVENT_UPLOAD_PREFIX="events/"
+  -e LANGFUSE_S3_MEDIA_UPLOAD_BUCKET="langfuse"
+  -e LANGFUSE_S3_MEDIA_UPLOAD_REGION="auto"
+  -e LANGFUSE_S3_MEDIA_UPLOAD_ACCESS_KEY_ID="minio"
+  -e LANGFUSE_S3_MEDIA_UPLOAD_SECRET_ACCESS_KEY="miniosecret"
+  -e LANGFUSE_S3_MEDIA_UPLOAD_ENDPOINT="http://localhost:9000"
+  -e LANGFUSE_S3_MEDIA_UPLOAD_FORCE_PATH_STYLE=true
+  -e LANGFUSE_S3_MEDIA_UPLOAD_PREFIX="media/"
+  -e LANGFUSE_S3_BATCH_EXPORT_ENABLED=false
+  -e LANGFUSE_S3_BATCH_EXPORT_BUCKET="langfuse"
+  -e LANGFUSE_S3_BATCH_EXPORT_PREFIX="exports/"
+  -e LANGFUSE_S3_BATCH_EXPORT_REGION="auto"
+  -e LANGFUSE_S3_BATCH_EXPORT_ENDPOINT="http://localhost:9000"
+  -e LANGFUSE_S3_BATCH_EXPORT_EXTERNAL_ENDPOINT="$LANGFUSE_PUBLIC_URL"
+  -e LANGFUSE_S3_BATCH_EXPORT_ACCESS_KEY_ID="minio"
+  -e LANGFUSE_S3_BATCH_EXPORT_SECRET_ACCESS_KEY="miniosecret"
+  -e LANGFUSE_S3_BATCH_EXPORT_FORCE_PATH_STYLE=true
+  -e LANGFUSE_INGESTION_QUEUE_DELAY_MS=
+  -e LANGFUSE_INGESTION_CLICKHOUSE_WRITE_INTERVAL_MS=
+  -e REDIS_HOST="localhost"
+  -e REDIS_PORT=6379
+  -e REDIS_AUTH="myredissecret"
+  -e REDIS_TLS_ENABLED=false
+  -e REDIS_TLS_CA="/certs/ca.crt"
+  -e REDIS_TLS_CERT="/certs/redis.crt"
+  -e REDIS_TLS_KEY="/certs/redis.key"
+  -e EMAIL_FROM_ADDRESS=
+  -e SMTP_CONNECTION_URL=
+)
+
 # Prepare directories
-mkdir -p "$LANGFLOW_DATA_DIR" "$POSTGRES_DATA_DIR" "$USER_SYSTEMD_DIR"
+mkdir -p "$LANGFLOW_DATA_DIR" "$POSTGRES_DATA_DIR" \
+  "$LANGFUSE_CLICKHOUSE_DATA_DIR" "$LANGFUSE_CLICKHOUSE_LOGS_DIR" \
+  "$LANGFUSE_MINIO_DATA_DIR" "$USER_SYSTEMD_DIR"
 
 # Create pod if not yet existing
 if ! podman pod exists "$POD_NAME"; then
   podman pod create -n "$POD_NAME" \
-    -p "$HOST_LOCAL_IP:$LANGFLOW_HOST_PORT:$LANGFLOW_CONTAINER_PORT"
+    -p "$HOST_LOCAL_IP:$LANGFLOW_HOST_PORT:$LANGFLOW_CONTAINER_PORT" \
+    -p "$HOST_LOCAL_IP:$LANGFUSE_HOST_PORT:$LANGFUSE_CONTAINER_PORT"
   echo "Pod '$POD_NAME' created (rc=$?)"
 else
   echo "Pod '$POD_NAME' already exists."
@@ -36,8 +108,13 @@ fi
 # Remove any old containers (ignore errors if they don't exist)
 podman rm -f "$LANGFLOW_CTR_NAME" || true
 podman rm -f "$POSTGRES_CTR_NAME" || true
+podman rm -f "$LANGFUSE_WEB_CTR_NAME" || true
+podman rm -f "$LANGFUSE_WORKER_CTR_NAME" || true
+podman rm -f "$CLICKHOUSE_CTR_NAME" || true
+podman rm -f "$MINIO_CTR_NAME" || true
+podman rm -f "$REDIS_CTR_NAME" || true
 
-# Postgres container
+# Postgres container (shared by Langflow and Langfuse)
 podman run -d --name "$POSTGRES_CTR_NAME" --pod "$POD_NAME" \
   -e POSTGRES_USER=langflow \
   -e POSTGRES_PASSWORD=langflow \
@@ -45,14 +122,10 @@ podman run -d --name "$POSTGRES_CTR_NAME" --pod "$POD_NAME" \
   -v "$POSTGRES_DATA_DIR:/var/lib/postgresql/data:Z" \
   "$POSTGRES_IMAGE"
 echo "Container '$POSTGRES_CTR_NAME' started (rc=$?)"
-
-# Wait for Postgres to be ready before starting Langflow
-# This avoids initial connection-refused errors in Langflow logs
-# and ensures DB-backed features (like Global Variables) are
-# available as soon as Langflow starts.
-echo "Waiting for Postgres to be ready..."
+# Wait for Postgres to be ready before starting Langflow / Langfuse
+echo "Waiting for Postgres to be ready (pg_isready)..."
 for attempt in $(seq 1 30); do
-  if podman exec "$POSTGRES_CTR_NAME" pg_isready -U langflow -d langflow >/dev/null 2>&1; then
+  if podman exec "$POSTGRES_CTR_NAME" pg_isready -q >/dev/null 2>&1; then
     echo "Postgres is ready."
     break
   fi
@@ -63,9 +136,40 @@ for attempt in $(seq 1 30); do
   fi
 done
 
+# Ensure separate database for Langfuse in the shared Postgres instance
+echo "Ensuring 'langfuse' database exists for Langfuse..."
+if ! podman exec "$POSTGRES_CTR_NAME" psql -U langflow -d postgres -tc "SELECT 1 FROM pg_database WHERE datname = 'langfuse';" | grep -q 1; then
+  podman exec "$POSTGRES_CTR_NAME" psql -U langflow -d postgres -c "CREATE DATABASE langfuse OWNER langflow;"
+fi
+echo "Database 'langfuse' is ready for Langfuse."
+
+# Clickhouse container (Langfuse analytics store)
+podman run -d --name "$CLICKHOUSE_CTR_NAME" --pod "$POD_NAME" \
+  -e CLICKHOUSE_DB=default \
+  -e CLICKHOUSE_USER=clickhouse \
+  -e CLICKHOUSE_PASSWORD=clickhouse \
+  -v "$LANGFUSE_CLICKHOUSE_DATA_DIR:/var/lib/clickhouse:Z" \
+  -v "$LANGFUSE_CLICKHOUSE_LOGS_DIR:/var/log/clickhouse-server:Z" \
+  "$CLICKHOUSE_IMAGE"
+echo "Container '$CLICKHOUSE_CTR_NAME' started (rc=$?)"
+
+# Minio container (S3-compatible storage for Langfuse)
+podman run -d --name "$MINIO_CTR_NAME" --pod "$POD_NAME" \
+  -e MINIO_ROOT_USER=minio \
+  -e MINIO_ROOT_PASSWORD=miniosecret \
+  -v "$LANGFUSE_MINIO_DATA_DIR:/data:Z" \
+  "$MINIO_IMAGE" \
+  sh -c 'mkdir -p /data/langfuse && minio server --address ":9000" --console-address ":9001" /data'
+echo "Container '$MINIO_CTR_NAME' started (rc=$?)"
+
+# Redis container (queue/cache for Langfuse)
+podman run -d --name "$REDIS_CTR_NAME" --pod "$POD_NAME" \
+  "$REDIS_IMAGE" \
+  --requirepass myredissecret \
+  --maxmemory-policy noeviction
+echo "Container '$REDIS_CTR_NAME' started (rc=$?)"
+
 # Langflow container
-# In a Podman pod all containers share the same network namespace,
-# so Postgres is reachable via localhost instead of the container name.
 podman run -d --name "$LANGFLOW_CTR_NAME" --pod "$POD_NAME" \
   -e LANGFLOW_DATABASE_URL=postgresql://langflow:langflow@localhost:5432/langflow \
   -e LANGFLOW_CONFIG_DIR=/app/langflow \
@@ -78,6 +182,28 @@ podman run -d --name "$LANGFLOW_CTR_NAME" --pod "$POD_NAME" \
   "$LANGFLOW_IMAGE"
 echo "Container '$LANGFLOW_CTR_NAME' started (rc=$?)"
 
+# Langfuse worker container
+podman run -d --name "$LANGFUSE_WORKER_CTR_NAME" --pod "$POD_NAME" \
+  "${LANGFUSE_COMMON_ENV[@]}" \
+  "$LANGFUSE_WORKER_IMAGE"
+echo "Container '$LANGFUSE_WORKER_CTR_NAME' started (rc=$?)"
+
+# Langfuse web container (exposed on host port 8091 -> 3000)
+podman run -d --name "$LANGFUSE_WEB_CTR_NAME" --pod "$POD_NAME" \
+  "${LANGFUSE_COMMON_ENV[@]}" \
+  -e NEXTAUTH_SECRET="qwexczutbewrgerznupvemqyw" \
+  -e LANGFUSE_INIT_ORG_ID= \
+  -e LANGFUSE_INIT_ORG_NAME= \
+  -e LANGFUSE_INIT_PROJECT_ID= \
+  -e LANGFUSE_INIT_PROJECT_NAME= \
+  -e LANGFUSE_INIT_PROJECT_PUBLIC_KEY= \
+  -e LANGFUSE_INIT_PROJECT_SECRET_KEY= \
+  -e LANGFUSE_INIT_USER_EMAIL= \
+  -e LANGFUSE_INIT_USER_NAME= \
+  -e LANGFUSE_INIT_USER_PASSWORD= \
+  "$LANGFUSE_WEB_IMAGE"
+echo "Container '$LANGFUSE_WEB_CTR_NAME' started (rc=$?)"
+
 # Generate systemd service files
 cd "$USER_SYSTEMD_DIR"
 podman generate systemd --name --new --files "$POD_NAME"
@@ -103,11 +229,11 @@ echo "Enabled systemd service pod-${POD_NAME}.service (rc=$?)"
 echo "To view status: systemctl --user status pod-${POD_NAME}.service"
 echo "To view logs: journalctl --user -u pod-${POD_NAME}.service -f"
 
-# Wait for API readiness
+# Wait for Langflow API readiness
 CHECK_URL="http://$HOST_LOCAL_IP:$LANGFLOW_HOST_PORT"
 for attempt in $(seq 1 30); do
   if curl -fsS "$CHECK_URL" >/dev/null 2>&1; then
-    echo "Langflow REST API is reachable at http://$HOST_LOCAL_IP:$LANGFLOW_HOST_PORT."
+    echo "Langflow Web UI is reachable at http://$HOST_LOCAL_IP:$LANGFLOW_HOST_PORT."
     break
   fi
   sleep 2