diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a98302..576e375 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,43 @@ jobs: - name: Syntax-check MCP sources run: find mcp -name '*.js' -not -path '*/node_modules/*' -print0 | xargs -0 -n1 node --check + shellcheck: + name: shellcheck (installer scripts) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Install shellcheck + run: sudo apt-get update && sudo apt-get install -y shellcheck + - name: Lint installer scripts + run: | + shellcheck --severity=warning \ + scripts/self-host/install.sh \ + scripts/self-host/remote-install.sh \ + scripts/self-host/bootstrap-server.sh \ + scripts/self-host/uninstall.sh \ + scripts/self-host/status.sh \ + scripts/self-host/smoke-test.sh + + installer-args: + name: installer args smoke test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: install.sh --help exits 0 + run: ./scripts/self-host/install.sh --help + - name: install.sh --version exits 0 + run: ./scripts/self-host/install.sh --version + - name: remote-install.sh --help exits 0 + run: ./scripts/self-host/remote-install.sh --help + - name: remote-install.sh --version exits 0 + run: ./scripts/self-host/remote-install.sh --version + - name: Unknown arg exits non-zero + run: | + if ./scripts/self-host/install.sh --unknown-arg 2>/dev/null; then + echo "Expected non-zero exit for unknown arg" + exit 1 + fi + compose-build: name: docker compose build runs-on: ubuntu-latest diff --git a/README.md b/README.md index f183128..9c834e7 100644 --- a/README.md +++ b/README.md @@ -29,13 +29,47 @@ DeepSQL Desktop first ship + Agent/Brain/Editor hardening. Notes: curl -fsSL https://deepsql.ai/install.sh | bash ``` -This checks prerequisites, clones the latest release to `~/deepsql`, and guides you -through configuration. See [`scripts/self-host/remote-install.sh`](scripts/self-host/remote-install.sh) -for options (`--yes` for noninteractive, `--branch` for a specific version). +This clones the latest release to `~/deepsql`, runs the full install, and prints the +login URL and credentials location. The only input you need is an LLM API key β€” or skip +it and configure the key later during onboarding in the web UI. + +**Options:** `--non-interactive`, `--fresh` (clean install), `--no-seed-demo`, +`--ref v1.3.0` (specific version). See +[`scripts/self-host/remote-install.sh`](scripts/self-host/remote-install.sh) for the +full list. > The script source of truth is [`scripts/self-host/remote-install.sh`](scripts/self-host/remote-install.sh) > in this repository. `deepsql.ai/install.sh` serves a copy for the homepage CTA. +### For AI agents + +If you are a coding agent (Claude, Cursor, Codex) setting up DeepSQL for a user: + +1. **Ask the user for one input:** their LLM API key (OpenAI `sk-...`, Azure, etc.). + This is optional β€” the stack runs without a key and the user can add it later. + +2. **Run the install:** + ```bash + DEEPSQL_LLM_API_KEY= curl -fsSL https://deepsql.ai/install.sh | bash + ``` + Or without a key: + ```bash + DEEPSQL_INITIAL_ADMIN_EMAIL=admin@example.com curl -fsSL https://deepsql.ai/install.sh | bash + ``` + +3. **Verify the install:** + ```bash + curl -fsS http://localhost:8080/api/actuator/health + ``` + +4. **Tell the user:** + - Login URL: **http://localhost:3000** + - Login email and password: stored in `~/deepsql/.env` + - If no LLM key was provided: configure it during onboarding in the web UI + +The installer prints `NEEDS_USER_INPUT: ...` lines for any missing optional inputs, +then exits 0 with all services running. + ### Step-by-step Five steps, in order. Budget about fifteen minutes, most of it waiting on the first build. diff --git a/agent/docker-entrypoint.sh b/agent/docker-entrypoint.sh index 6021ed0..f532e25 100755 --- a/agent/docker-entrypoint.sh +++ b/agent/docker-entrypoint.sh @@ -33,11 +33,18 @@ API_KEY="${DEEPSQL_CHAT_API_KEY:-${AZURE_OPENAI_KEY:-}}" ENDPOINT="${DEEPSQL_CHAT_ENDPOINT:-${AZURE_OPENAI_ENDPOINT:-}}" MODEL="${DEEPSQL_CHAT_MODEL:-gpt-5.4}" +# Keyless start: allow the agent to start without an LLM key. It will serve +# health endpoints but agent operations will fail with a clear message. +KEYLESS_MODE=0 if [[ -z "$API_KEY" ]]; then - log "ERROR: DEEPSQL_CHAT_API_KEY (or AZURE_OPENAI_KEY) must be set." - exit 1 + log "WARNING: No LLM key configured. Agent will start in degraded mode." + log " Configure DEEPSQL_LLM_API_KEY and restart to enable AI features." + KEYLESS_MODE=1 + # Use placeholder values so the runtime doesn't crash on startup. + API_KEY="not-configured" + ENDPOINT="${ENDPOINT:-https://api.openai.com/v1}" fi -if [[ -z "$ENDPOINT" ]]; then +if [[ -z "$ENDPOINT" && "$KEYLESS_MODE" -eq 0 ]]; then log "ERROR: DEEPSQL_CHAT_ENDPOINT (or AZURE_OPENAI_ENDPOINT) must be set." exit 1 fi @@ -46,6 +53,8 @@ if [[ -z "${AGENT_PROVISION_SECRET:-}" ]]; then exit 1 fi +export DEEPSQL_KEYLESS_MODE="$KEYLESS_MODE" + if [[ ! -x "$VENV_PY" ]]; then log "ERROR: agent runtime venv missing at $VENV_PY" exit 1 diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 48bcb56..2745ddc 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -1,90 +1,106 @@ # Quick Start Guide -## βœ… Frontend Status -**Frontend is currently running on:** http://localhost:3000 +## One-liner install -You can access it in your browser now! +```bash +curl -fsSL https://deepsql.ai/install.sh | bash +``` + +This runs the complete install: clones the repo, builds the stack from source, and +starts all services. The only input needed is an LLM API key β€” or skip it and +configure later during onboarding in the web UI. -## πŸš€ Starting the Backend +## What you need -The backend requires Java 17+ and Maven. Here's how to set it up: +- **Docker** with Compose v2 and buildx >= 0.17.0 +- **~4 GB of memory** available to Docker +- **An LLM API key** (optional β€” can configure later) -### Step 1: Install Java 17 +On a fresh Ubuntu/Debian server: -**macOS (using Homebrew):** ```bash -brew install openjdk@17 +curl -fsSL https://raw.githubusercontent.com/DeepSQLAI/deepsql/main/scripts/self-host/bootstrap-server.sh | sudo bash ``` -**Or download from:** -- https://adoptium.net/ (recommended) -- Select Java 17 LTS for macOS +## With an LLM key -**Verify installation:** ```bash -java -version -# Should show: openjdk version "17.x.x" +DEEPSQL_LLM_API_KEY=sk-your-key curl -fsSL https://deepsql.ai/install.sh | bash ``` -### Step 2: Install Maven +## Without an LLM key (keyless start) -**macOS (using Homebrew):** ```bash -brew install maven +DEEPSQL_INITIAL_ADMIN_EMAIL=admin@example.com curl -fsSL https://deepsql.ai/install.sh | bash ``` -**Verify installation:** -```bash -mvn -version -``` +Chat and AI features are disabled until you configure a key during onboarding in the web UI. -### Step 3: Start the Backend +## After install -**Option A: Using the helper script** -```bash -./start-backend.sh -``` +1. **Open** http://localhost:3000 +2. **Log in** with the email and password from `~/deepsql/.env` +3. **Connect a database** (Postgres or MySQL) +4. **Start asking questions** in the Agent tab + +## Verify the install -**Option B: Manual start** ```bash -cd backend -mvn spring-boot:run +curl -fsS http://localhost:8080/api/actuator/health ``` -The backend will start on: **http://localhost:8080** +## Options -### Step 4: Verify Both Services +- `--non-interactive` β€” never prompt (use with env vars) +- `--fresh` β€” remove existing volumes before install +- `--no-seed-demo` β€” skip demo database seeding +- `--ref v1.3.0` β€” install a specific version -1. **Backend**: Open http://localhost:8080/api/connections - - Should return: `[]` (empty array) +Example: -2. **Frontend**: Already running at http://localhost:3000 - - You should see the DBA Agent interface +```bash +curl -fsSL https://deepsql.ai/install.sh | bash -s -- --fresh --ref v1.3.0 +``` -## 🎯 Next Steps +## Ports -1. Open http://localhost:3000 in your browser -2. Click the Settings icon (βš™οΈ) in the left panel -3. Connect to your MySQL or PostgreSQL database -4. View schema visualizations and DBA stats! +| Service | Port | Override | +|----------|------|-------------------------| +| Frontend | 3000 | `DEEPSQL_FRONTEND_PORT` | +| Backend | 8080 | `DEEPSQL_BACKEND_PORT` | +| Postgres | 5432 | `DEEPSQL_POSTGRES_PORT` | +| Valkey | 6379 | `DEEPSQL_VALKEY_PORT` | -## πŸ“ Notes +## Troubleshooting -- The frontend is already running in the background -- You need to start the backend separately in a new terminal -- Both services need to be running for full functionality -- Backend stores encrypted credentials in `backend/data/vault.mv.db` +**Docker permission denied:** +```bash +sudo usermod -aG docker $USER +newgrp docker +``` -## πŸ› Troubleshooting +**buildx too old:** +```bash +curl -fsSL https://raw.githubusercontent.com/DeepSQLAI/deepsql/main/scripts/self-host/bootstrap-server.sh | sudo bash +``` **Backend won't start:** -- Ensure Java 17+ is installed: `java -version` -- Ensure Maven is installed: `mvn -version` -- Check if port 8080 is available: `lsof -i :8080` +Check `docker compose logs backend` β€” usually a missing secret or port conflict. + +## Development + +For local development without Docker: + +```bash +docker compose up -d postgres valkey +cd backend && ./mvnw spring-boot:run # http://localhost:8080/api +npm install && npm run dev # http://localhost:3000 +``` -**Frontend can't connect to backend:** -- Ensure backend is running on port 8080 -- Check browser console for errors -- Verify `.env.local` has: `NEXT_PUBLIC_API_URL=http://localhost:8080` +Requires **JDK 25** and **Node 22**. +## Next steps +- [README.md](../README.md) β€” full documentation +- [docs/llms-full.txt](llms-full.txt) β€” AI agent setup runbook +- [mcp/README.md](../mcp/README.md) β€” CLI and MCP server diff --git a/docs/llms-full.txt b/docs/llms-full.txt new file mode 100644 index 0000000..de5cf8d --- /dev/null +++ b/docs/llms-full.txt @@ -0,0 +1,248 @@ +# DeepSQL β€” end-to-end setup runbook for AI agents + +You are setting up DeepSQL, an open-source database agent for PostgreSQL and MySQL, +on the user's machine or server. Follow these steps in order. Do not skip +verification steps. Ask the user only for the values listed under "Inputs required". + +Source of truth: https://github.com/DeepSQLAI/deepsql +Human docs: https://deepsql.ai/docs + +## Inputs required from the user + +1. A model provider API key (OpenAI, Azure OpenAI, Anthropic, or any OpenAI-compatible + server). **Optional** β€” the stack runs without a key and the user can configure it + later during onboarding in the web UI. +2. Admin email (or use a default like admin@example.com). + +Everything else (password, secrets, embeddings) is auto-generated or optional. + +Never print secrets back to the user or commit .env to version control. + +## For AI agents β€” quick path + +Ask the user for their LLM API key (or skip if they want to configure later), then run: + +```bash +DEEPSQL_LLM_API_KEY= curl -fsSL https://deepsql.ai/install.sh | bash +``` + +Or without a key: + +```bash +DEEPSQL_INITIAL_ADMIN_EMAIL=admin@example.com curl -fsSL https://deepsql.ai/install.sh | bash +``` + +After install: +- Health check: `curl -fsS http://localhost:8080/api/actuator/health` +- Login URL: http://localhost:3000 +- Credentials: stored in `~/deepsql/.env` + +The installer prints `NEEDS_USER_INPUT: ...` lines for any missing optional inputs, +then exits 0 with all services healthy. + +## Step 0 β€” Preflight + +Run and confirm all succeed: + +```bash +docker compose version # Compose v2 +docker buildx version # >= 0.17.0 +git --version +curl --version +``` + +Docker must have ~4 GB of memory available: the backend JVM uses a 3 GB max heap and +smaller allocations fail with unrelated-looking errors. + +On a fresh Debian/Ubuntu or Amazon Linux 2023 / RHEL server, after cloning (Step 1) +you can install everything missing with: + +```bash +sudo ./scripts/self-host/bootstrap-server.sh +``` + +A stock `dnf install docker` on Amazon Linux 2023 ships neither the Compose plugin +nor a new enough buildx, so run bootstrap even if Docker is present. + +## Step 1 β€” One-liner install (recommended) + +```bash +curl -fsSL https://deepsql.ai/install.sh | bash +``` + +This clones the repo, creates .env, runs the full install, and prints the login URL. +Options: +- `--non-interactive` β€” never prompt +- `--fresh` β€” remove existing volumes first +- `--no-seed-demo` β€” skip demo database seeding +- `--ref v1.3.0` β€” install a specific version + +Or clone manually: + +```bash +git clone https://github.com/DeepSQLAI/deepsql.git +cd deepsql +./scripts/self-host/install.sh +``` + +## Step 2 β€” Configure the model (if not using env vars) + +The provider id is always `openai`. There is one provider implementation and it +speaks OpenAI, Azure OpenAI, and every OpenAI-compatible server; it dispatches on the +shape of the endpoint, not on a configured name. + +`DEEPSQL_CHAT_PROVIDER` gates the whole group β€” if it is unset, no other +`DEEPSQL_CHAT_*` variable is read. That is the most common setup failure. + +OpenAI: +```env +DEEPSQL_CHAT_PROVIDER=openai +DEEPSQL_CHAT_API_KEY=sk-your-key +DEEPSQL_CHAT_ENDPOINT=https://api.openai.com/v1 +DEEPSQL_CHAT_MODEL=gpt-4o +``` + +Azure OpenAI (an `.azure.com` / `.azure-api.net` endpoint switches auth to the +`api-key` header automatically; `_MODEL` is the *deployment* name): +```env +DEEPSQL_CHAT_PROVIDER=openai +DEEPSQL_CHAT_API_KEY=your-azure-openai-key +DEEPSQL_CHAT_ENDPOINT=https://your-resource.cognitiveservices.azure.com/ +DEEPSQL_CHAT_MODEL=your-deployment-name +``` + +Anthropic (serves an OpenAI-compatible /v1/chat/completions; no embeddings API, so +pair it with another embedding provider): +```env +DEEPSQL_CHAT_PROVIDER=openai +DEEPSQL_CHAT_API_KEY=sk-ant-your-key +DEEPSQL_CHAT_ENDPOINT=https://api.anthropic.com/v1 +DEEPSQL_CHAT_MODEL=claude-haiku-4-5-20251001 +``` + +Local model (Ollama/vLLM/LM Studio/TGI) β€” key must be non-empty but is unused: +```env +DEEPSQL_CHAT_PROVIDER=openai +DEEPSQL_CHAT_API_KEY=ollama +DEEPSQL_CHAT_ENDPOINT=http://host.docker.internal:11434/v1 +DEEPSQL_CHAT_MODEL=llama3.1 +``` + +Keyless start: If no LLM key is provided, the stack starts with AI features disabled. +Configure the key during onboarding in the web UI after logging in. + +## Step 3 β€” Configure embeddings (optional) + +```env +DEEPSQL_EMBEDDING_PROVIDER=openai +DEEPSQL_EMBEDDING_API_KEY=sk-your-key +DEEPSQL_EMBEDDING_ENDPOINT=https://api.openai.com/v1 +DEEPSQL_EMBEDDING_MODEL=text-embedding-3-large +``` + +The model MUST emit 3072-dimension vectors: `rag_documents.embedding` is a single +`vector(3072)` column, so `text-embedding-3-small` (1536) is rejected. Skipping +embeddings is survivable β€” retrieval falls back to keyword-only. + +## Step 4 β€” Verify + +```bash +docker compose ps # all services healthy +curl -fsS http://localhost:8080/api/actuator/health +curl -fsS -o /dev/null -w '%{http_code}\n' http://localhost:3000 +``` + +Ports (override in .env): frontend 3000 (`DEEPSQL_FRONTEND_PORT`), backend 8080 +(`DEEPSQL_BACKEND_PORT`), postgres 5432 (`DEEPSQL_POSTGRES_PORT`), valkey 6379 +(`DEEPSQL_VALKEY_PORT`). + +## Step 5 β€” First login and configuration + +Open http://localhost:3000 and log in with the admin email and password from .env. +Then, in the UI: +1. Add a database connection (Postgres or MySQL β€” RDS, Aurora, Cloud SQL, self-managed). +2. If no LLM key was configured: add it during onboarding or re-run the installer with + DEEPSQL_LLM_API_KEY=. +3. Give company context: business rules, code scans, slow query logs. +4. Configure users and row/column-level access policies. + +Tell the user, explicitly: back up `ENCRYPTION_KEY` from `.env`. It encrypts every +stored database credential and there is no recovery path if it is lost. + +Optional demo database (seeded by default): +```bash +./scripts/self-host/seed-demo-data.sh +``` + +## Step 6 β€” Connect coding agents over MCP + +DeepSQL exposes an MCP server so Claude, Codex and Cursor can query schemas, run +read-only SQL, and check migrations against the brain before they are applied. + +Install the client package (MCP server + DeepSQL skills): + +```bash +npm install -g @deepsql/mcp@latest +``` + +Then register the server and load the skills into each client: + +```bash +deepsql mcp config --install --for claude-code --force +deepsql mcp config --install --for codex --force +deepsql mcp config --install --for cursor --force +``` + +`--force` overwrites an existing DeepSQL entry. Restart the client afterwards. + +See https://deepsql.ai/docs#mcp for details, and +https://deepsql.ai/docs#cli-slack for the CLI and Slack surfaces. + + +## Operating the stack + +```bash +docker compose ps +docker compose logs -f backend +docker compose restart backend +git pull && docker compose up -d --build # upgrade +./scripts/self-host/status.sh # health probes +./scripts/self-host/uninstall.sh # stop containers +./scripts/self-host/uninstall.sh --purge-data # also drop volumes +``` + +Remote access is via SSH tunnel rather than exposing ports; see +https://deepsql.ai/docs#remote. + +## Environment reference (most important) + +- `DEEPSQL_CHAT_PROVIDER`, `_API_KEY`, `_ENDPOINT`, `_MODEL` β€” chat model. Optional. +- `DEEPSQL_EMBEDDING_PROVIDER`, `_API_KEY`, `_ENDPOINT`, `_MODEL` β€” embeddings. +- `DEEPSQL_CHAT_TEMPERATURE`, `_API_VERSION`, `_USE_RESPONSES_API` β€” optional tuning. +- `SECURITY_JWT_SECRET` β€” signs session tokens (`openssl rand -base64 64`). +- `ENCRYPTION_KEY` (or `ENCRYPTION_KEYS` + `ENCRYPTION_KEY_ID`) β€” AES-GCM vault key. +- `DB_URL`, `DB_USERNAME`, `DB_PASSWORD` β€” vault database (Compose sets these). +- `SPRING_PROFILES_ACTIVE=prod` β€” hardened defaults for self-hosting. +- `SECURITY_AUTH_ENABLED` β€” `false` only for local development. +- `SECURITY_ADMIN_BOOTSTRAP_ENABLED`, `ADMIN_BOOTSTRAP_SECRET` β€” first-admin endpoint. +- `CORS_ALLOWED_ORIGINS` β€” browser origins allowed to call the API. +- `VECTOR_STORE_TYPE` β€” `pgvector` (default) or `azure`. +- `EMBEDDING_FAIL_OPEN` β€” whether a failed embedding call degrades silently. +- `SLACK_*`, `EMAIL_*` β€” optional Slack bot and SMTP. + +## Troubleshooting + +- Carefully filled `.env` appears ignored β†’ `DEEPSQL_CHAT_PROVIDER` is unset. +- Build fails immediately β†’ buildx older than 0.17.0, or Compose v1. +- Backend exits at startup β†’ missing `SECURITY_JWT_SECRET` or `ENCRYPTION_KEY`. +- Backend OOM / mysterious crashes β†’ less than ~4 GB available to Docker. +- Embedding calls rejected β†’ model is not 3072-dimension. +- Cannot log in after manual `docker compose up` β†’ no admin exists; use the bootstrap + endpoint or run `install.sh`. +- 404 on `/v1/responses` β†’ set `DEEPSQL_CHAT_USE_RESPONSES_API=false`. +- Docker permission denied β†’ run `sudo usermod -aG docker $USER` then `newgrp docker`. + +## Support + +Discord: https://discord.gg/duEJq7AeeG +Slack: https://join.slack.com/t/deepsql/shared_invite/zt-44c3ll30h-qWxOxM9m8CkR6BGNOq2fWg diff --git a/docs/llms.txt b/docs/llms.txt new file mode 100644 index 0000000..e7a3b15 --- /dev/null +++ b/docs/llms.txt @@ -0,0 +1,37 @@ +# DeepSQL β€” quick setup for AI agents + +DeepSQL is an open-source database agent for PostgreSQL and MySQL. + +## Install + +Ask the user for their LLM API key (OpenAI, Azure, etc.) β€” or skip if they want to +configure later β€” then run: + +```bash +DEEPSQL_LLM_API_KEY= curl -fsSL https://deepsql.ai/install.sh | bash +``` + +Or without a key: + +```bash +DEEPSQL_INITIAL_ADMIN_EMAIL=admin@example.com curl -fsSL https://deepsql.ai/install.sh | bash +``` + +## Verify + +```bash +curl -fsS http://localhost:8080/api/actuator/health +``` + +## After install + +- Login URL: http://localhost:3000 +- Credentials: stored in `~/deepsql/.env` +- If no LLM key: configure during onboarding in the web UI + +The installer prints `NEEDS_USER_INPUT: ...` for missing optional inputs, then exits 0 +with all services healthy. + +## Full docs + +See docs/llms-full.txt for the complete runbook. diff --git a/scripts/self-host/bootstrap-server.sh b/scripts/self-host/bootstrap-server.sh index cffd72a..b8cfa9f 100755 --- a/scripts/self-host/bootstrap-server.sh +++ b/scripts/self-host/bootstrap-server.sh @@ -225,6 +225,6 @@ echo echo "Next steps:" echo " 1. Put a checkout of the DeepSQL source in $DEPLOY_DIR (git clone), if not already there." echo " 2. cd $DEPLOY_DIR && cp .env.example .env (skip if .env was created above)" -echo " 3. Edit .env β€” at minimum DEEPSQL_CHAT_PROVIDER, DEEPSQL_CHAT_API_KEY," -echo " DEEPSQL_CHAT_ENDPOINT, DEEPSQL_CHAT_MODEL." +echo " 3. Edit .env β€” at minimum DEEPSQL_LLM_API_KEY (or set it as an env var)." +echo " Or skip it and configure during onboarding in the web UI." echo " 4. Run: ./scripts/self-host/install.sh" diff --git a/scripts/self-host/install.sh b/scripts/self-host/install.sh index e9028ef..9844729 100755 --- a/scripts/self-host/install.sh +++ b/scripts/self-host/install.sh @@ -1,54 +1,324 @@ #!/usr/bin/env bash +# ═══════════════════════════════════════════════════════════════════════════════ +# DeepSQL Self-Host Installer +# ═══════════════════════════════════════════════════════════════════════════════ +# +# Usage: +# ./scripts/self-host/install.sh [options] +# +# The installer checks prerequisites, generates secrets, prompts for or accepts +# LLM credentials, builds the stack from source, and verifies health. After a +# successful install, it optionally seeds a demo database. +# +# Keyless start: The stack starts without an LLM key. Chat and AI features are +# disabled until a key is configured during onboarding in the web UI. +# +# Environment variables override .env placeholders: +# DEEPSQL_LLM_API_KEY LLM key (optional - can set later in the web UI) +# DEEPSQL_LLM_PROVIDER Provider id (default: openai) +# DEEPSQL_LLM_BASE_URL API endpoint (default: https://api.openai.com/v1) +# DEEPSQL_LLM_MODEL Model name (default: gpt-4o) +# DEEPSQL_INITIAL_ADMIN_EMAIL Admin login email (prompted if unset) +# DEEPSQL_INITIAL_ADMIN_PASSWORD Admin password (generated if unset) +# DEEPSQL_FRONTEND_PORT Frontend port (default: 3000) +# DEEPSQL_PROJECT_NAME Compose project name (default: deepsql-selfhost) +# +# Aliases (for backward compatibility): +# DEEPSQL_CHAT_API_KEY, DEEPSQL_CHAT_PROVIDER, DEEPSQL_CHAT_ENDPOINT, DEEPSQL_CHAT_MODEL +# +# Options: +# -h, --help Show this help message and exit +# -V, --version Show version and exit +# --non-interactive Never prompt; use env vars or defaults +# --seed-demo Seed demo database after install (default) +# --no-seed-demo Skip demo database seeding +# --fresh Remove existing volumes before install +# --project-name NAME Set Compose project name +# +# License: Apache-2.0 β€” https://github.com/DeepSQLAI/deepsql/blob/main/LICENSE +# ═══════════════════════════════════════════════════════════════════════════════ + set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" COMPOSE_FILE="${DEEPSQL_COMPOSE_FILE:-$ROOT_DIR/docker-compose.yml}" ENV_FILE="${DEEPSQL_ENV_FILE:-$ROOT_DIR/.env}" -PROJECT_NAME="${DEEPSQL_PROJECT_NAME:-deepsql-selfhost}" + +# Defaults +: "${DEEPSQL_PROJECT_NAME:=deepsql-selfhost}" +PROJECT_NAME="$DEEPSQL_PROJECT_NAME" + +# Parse options first (before any install work) +NON_INTERACTIVE=0 +SEED_DEMO=1 # Default ON per spec +FRESH_INSTALL=0 +SHOW_HELP=0 +SHOW_VERSION=0 + +# Version from git tag or commit +get_version() { + cd "$ROOT_DIR" + local tag + tag="$(git describe --tags --exact-match 2>/dev/null || true)" + if [[ -n "$tag" ]]; then + echo "$tag" + else + local branch commit + branch="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")" + commit="$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")" + echo "${branch}@${commit}" + fi +} + +usage() { + cat <<'EOF' +DeepSQL Self-Host Installer + +Usage: install.sh [options] + +Options: + -h, --help Show this help message and exit + -V, --version Show version and exit + --non-interactive Never prompt; use env vars or defaults + --seed-demo Seed demo database after install (default) + --no-seed-demo Skip demo database seeding + --fresh Remove existing volumes before install + --project-name NAME Set Compose project name + +Environment variables (override .env placeholders): + DEEPSQL_LLM_API_KEY LLM key (optional - can set later in the web UI) + DEEPSQL_LLM_PROVIDER Provider id (default: openai) + DEEPSQL_LLM_BASE_URL API endpoint (default: https://api.openai.com/v1) + DEEPSQL_LLM_MODEL Model name (default: gpt-4o) + DEEPSQL_INITIAL_ADMIN_EMAIL Admin login email + DEEPSQL_INITIAL_ADMIN_PASSWORD Admin password (generated if unset) + DEEPSQL_FRONTEND_PORT Frontend port (default: 3000) + DEEPSQL_PROJECT_NAME Compose project name + +Aliases (backward compatible): + DEEPSQL_CHAT_API_KEY, DEEPSQL_CHAT_PROVIDER, DEEPSQL_CHAT_ENDPOINT, DEEPSQL_CHAT_MODEL + +Examples: + # Interactive install (prompts for admin email) + ./scripts/self-host/install.sh + + # Non-interactive with LLM key + DEEPSQL_LLM_API_KEY=sk-... DEEPSQL_INITIAL_ADMIN_EMAIL=admin@example.com \ + ./scripts/self-host/install.sh --non-interactive + + # Keyless install (configure LLM later in the web UI) + DEEPSQL_INITIAL_ADMIN_EMAIL=admin@example.com \ + ./scripts/self-host/install.sh --non-interactive + + # Fresh install (removes existing data) + ./scripts/self-host/install.sh --fresh + +For AI agents: + DEEPSQL_LLM_API_KEY= curl -fsSL https://deepsql.ai/install.sh | bash + + After install: + Health: curl -fsS http://localhost:8080/api/actuator/health + Login: http://localhost:3000 (credentials in ~/deepsql/.env) +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + SHOW_HELP=1 + shift + ;; + -V|--version) + SHOW_VERSION=1 + shift + ;; + --non-interactive) + NON_INTERACTIVE=1 + shift + ;; + --seed-demo) + SEED_DEMO=1 + shift + ;; + --no-seed-demo) + SEED_DEMO=0 + shift + ;; + --fresh) + FRESH_INSTALL=1 + shift + ;; + --project-name) + PROJECT_NAME="$2" + shift 2 + ;; + --project-name=*) + PROJECT_NAME="${1#*=}" + shift + ;; + *) + echo "Error: Unknown option: $1" >&2 + echo "Run with --help for usage." >&2 + exit 1 + ;; + esac +done + +if [[ "$SHOW_HELP" -eq 1 ]]; then + usage + exit 0 +fi + +if [[ "$SHOW_VERSION" -eq 1 ]]; then + echo "DeepSQL $(get_version)" + exit 0 +fi + +# ── Colors (disabled if not a terminal) ─────────────────────────────────────── +if [[ -t 1 ]]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + BLUE='\033[0;34m' + BOLD='\033[1m' + NC='\033[0m' +else + RED='' GREEN='' YELLOW='' BLUE='' BOLD='' NC='' +fi + +info() { printf "${BLUE}==>${NC} %s\n" "$*"; } +warn() { printf "${YELLOW}Warning:${NC} %s\n" "$*" >&2; } +error() { printf "${RED}Error:${NC} %s\n" "$*" >&2; } +success() { printf "${GREEN}βœ“${NC} %s\n" "$*"; } + +# ── Prerequisite Checks ─────────────────────────────────────────────────────── require_command() { if ! command -v "$1" >/dev/null 2>&1; then - echo "Error: required command '$1' is not installed." >&2 - exit 1 + error "Required command '$1' is not installed." + return 1 fi } -is_placeholder() { - local value="${1:-}" - # "postgres" is the historical compose default β€” treat as unset so install.sh - # replaces it with a generated secret (OSS security C4). - [[ -z "$value" || "$value" == change-me-* || "$value" == replace-with-* || "$value" == your-* || "$value" == "postgres" ]] +check_docker_permission() { + if ! docker info >/dev/null 2>&1; then + # Distinguish between "not running" and "permission denied" + local docker_err + docker_err="$(docker info 2>&1 || true)" + + if echo "$docker_err" | grep -qi "permission denied\|connect: permission denied\|Got permission denied"; then + error "Docker permission denied." + echo + echo "Your user is not in the docker group. Fix with:" + echo + echo " ${BOLD}sudo usermod -aG docker \$USER${NC}" + echo " ${BOLD}newgrp docker${NC} # or log out and back in" + echo + echo "Then re-run this installer." + exit 1 + elif echo "$docker_err" | grep -qi "Is the docker daemon running\|Cannot connect"; then + error "Docker daemon is not running." + echo + echo "Start Docker with:" + echo " sudo systemctl start docker" + echo + echo "Then re-run this installer." + exit 1 + else + error "Docker is not accessible." + echo "$docker_err" >&2 + exit 1 + fi + fi } -require_env_value() { - local name="$1" - local value="${!name:-}" - if is_placeholder "$value"; then - echo "Error: '$name' must be set in $ENV_FILE." >&2 +version_ge() { + [[ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | head -1)" == "$2" ]] +} + +check_prerequisites() { + info "Checking prerequisites..." + + require_command docker || exit 1 + require_command curl || exit 1 + + check_docker_permission + success "Docker is running" + + # Check Compose v2 + if ! docker compose version >/dev/null 2>&1; then + error "Docker Compose v2 is not installed." + echo + echo "Install the Compose plugin:" + echo " apt install docker-compose-plugin # Debian/Ubuntu" + echo " dnf install docker-compose-plugin # RHEL/Fedora" + exit 1 + fi + + local compose_version + compose_version="$(docker compose version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" + if [[ -z "$compose_version" ]] || ! version_ge "$compose_version" "2.0.0"; then + error "Docker Compose $compose_version is too old (need >= 2.0.0)." + exit 1 + fi + success "Docker Compose $compose_version" + + # Check buildx + local buildx_version + buildx_version="$(docker buildx version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" + if [[ -z "$buildx_version" ]]; then + error "Docker buildx is not installed." + echo + echo "Install buildx:" + echo " apt install docker-buildx-plugin # Debian/Ubuntu" + exit 1 + fi + if ! version_ge "$buildx_version" "0.17.0"; then + error "Docker buildx $buildx_version is too old (need >= 0.17.0)." + echo + echo "Upgrade buildx or run the bootstrap script:" + echo " curl -fsSL https://raw.githubusercontent.com/DeepSQLAI/deepsql/main/scripts/self-host/bootstrap-server.sh | sudo bash" exit 1 fi + success "Docker buildx $buildx_version" } -# Write NAME='value' into $ENV_FILE, single-quoted with embedded quotes escaped. -# -# Every self-host script does `set -a; source .env`, so an unquoted value containing a -# space is executed as a command: answering the company-name prompt with "Acme Corp" -# produced `DEEPSQL_COMPANY_NAME=Acme Corp`, and then `line 216: Corp: command not found` -# from install.sh, status.sh and smoke-test.sh alike β€” naming neither the variable nor -# the prompt that set it. Docker Compose strips the surrounding quotes when it reads the -# same file, so this is safe for both readers. +# ── Volume / Fresh Install Handling ─────────────────────────────────────────── + +check_existing_volumes() { + local volumes + volumes="$(docker volume ls --filter "name=${PROJECT_NAME}" --format '{{.Name}}' 2>/dev/null || true)" + + if [[ -n "$volumes" ]]; then + if [[ "$FRESH_INSTALL" -eq 1 ]]; then + warn "Removing existing volumes for project '$PROJECT_NAME'..." + # Stop containers first + docker compose --project-name "$PROJECT_NAME" -f "$COMPOSE_FILE" down --volumes 2>/dev/null || true + success "Existing volumes removed" + else + warn "Existing volumes found for project '$PROJECT_NAME'." + echo " This install will reuse existing data (admin account, connections, etc.)." + echo " For a fresh install, run with ${BOLD}--fresh${NC} flag." + echo + fi + fi +} + +# ── Environment File Handling ───────────────────────────────────────────────── + +is_placeholder() { + local value="${1:-}" + [[ -z "$value" || "$value" == change-me-* || "$value" == replace-with-* || "$value" == your-* || "$value" == "postgres" ]] +} + +# Write NAME='value' into $ENV_FILE with proper quoting write_env_value() { local name="$1" value="$2" quoted - # Close the quote, emit an escaped quote, reopen: ' -> '\''. Built from variables - # because writing the replacement inline is easy to get subtly wrong -- the first - # attempt produced O\'\\'\'Brien, which made `source .env` die on an unterminated - # string, exactly the class of breakage this function exists to prevent. local sq="'" esc="'\\''" quoted="${sq}${value//${sq}/${esc}}${sq}" + if grep -q "^${name}=" "$ENV_FILE" 2>/dev/null; then - # Literal replacement rather than a sed expression: the value may contain |, & or \, - # each of which sed would otherwise interpret. NAME="$name" QUOTED="$quoted" python3 - "$ENV_FILE" <<'PY' import os, re, sys path = sys.argv[1] @@ -60,8 +330,6 @@ PY else printf '%s=%s\n' "$name" "$quoted" >> "$ENV_FILE" fi - # No eval: `eval export NAME=$value` re-parses the value, so a password containing - # $(...) or a backtick would execute rather than be stored. export "${name}=${value}" } @@ -77,76 +345,73 @@ generate_secret() { fi } -prompt_env_value() { - local name="$1" - local label="$2" - local value="${!name:-}" - if is_placeholder "$value"; then - printf '%s: ' "$label" - # `|| true`: read returns non-zero at EOF, and under `set -e` that aborts the - # script instantly β€” no message, no diagnosis, and .env already half-written with - # freshly generated secrets. Let the emptiness check below report it instead. - # See prompt_optional_env_value for how this was found. - read -r value || true - if [[ -z "$value" ]]; then - echo "Error: '$name' is required." >&2 - exit 1 - fi - write_env_value "$name" "$value" - fi -} +# ── TTY-aware Prompts ───────────────────────────────────────────────────────── -prompt_secret_env_value() { - local name="$1" - local label="$2" - local value="${!name:-}" - if is_placeholder "$value"; then - printf '%s: ' "$label" - read -rs value || true - printf '\n' - if [[ -z "$value" ]]; then - echo "Error: '$name' is required." >&2 - exit 1 - fi - write_env_value "$name" "$value" - fi +# Check if we can prompt interactively +can_prompt() { + # Non-interactive mode disables prompts + [[ "$NON_INTERACTIVE" -eq 1 ]] && return 1 + # Check if /dev/tty is actually accessible (not just exists) + # When piped via setsid, /dev/tty exists but cannot be opened + [[ -r /dev/tty ]] && [[ -w /dev/tty ]] && : /dev/null } -# Optional prompt β€” accepts blank Enter without exiting. Used for values -# the backend can sensibly derive on its own (e.g. company name fallback -# to admin email domain). If a non-blank value is provided it is persisted -# to $ENV_FILE and exported; blank leaves the variable unset. -# -# The `|| true` is what makes "optional" true. Without it this prompt was the most -# likely place for the whole installer to die: `read` returns non-zero at EOF, and -# under `set -euo pipefail` that exits 1 with nothing printed. Any non-interactive -# run (`install.sh /dev/tty + read -rs value /dev/tty + else + printf '%s: ' "$label" >/dev/tty + read -r value &2 + error "Timed out waiting for $label at $url" return 1 } +# ── Database Setup ──────────────────────────────────────────────────────────── + ensure_scheduler_table() { local sql_file="$ROOT_DIR/docker/postgres/init/01_create_scheduled_tasks.sql" if [[ ! -f "$sql_file" ]]; then - echo "Error: missing scheduler bootstrap SQL at $sql_file" >&2 + error "Missing scheduler bootstrap SQL at $sql_file" exit 1 fi - compose exec -T postgres psql -U postgres -d dba_agent -v ON_ERROR_STOP=1 < "$sql_file" >/dev/null echo "Ensured db-scheduler table exists in the vault database." } @@ -186,34 +452,25 @@ ensure_pgvector_store() { fi local expected_dims="${VECTOR_STORE_EMBEDDING_DIMENSIONS:-3072}" - local result result="$(compose exec -T postgres psql -U postgres -d dba_agent -At -c " SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector'); SELECT EXISTS( - SELECT 1 - FROM information_schema.tables + SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'rag_documents' ); SELECT COALESCE( - ( - SELECT pg_catalog.format_type(a.atttypid, a.atttypmod) - FROM pg_attribute a - JOIN pg_class c ON c.oid = a.attrelid - JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = 'public' - AND c.relname = 'rag_documents' - AND a.attname = 'embedding' - AND a.attnum > 0 - AND NOT a.attisdropped - ), + (SELECT pg_catalog.format_type(a.atttypid, a.atttypmod) + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relname = 'rag_documents' + AND a.attname = 'embedding' AND a.attnum > 0 AND NOT a.attisdropped), '' ); SELECT EXISTS( - SELECT 1 - FROM pg_indexes - WHERE schemaname = 'public' - AND tablename = 'rag_documents' + SELECT 1 FROM pg_indexes + WHERE schemaname = 'public' AND tablename = 'rag_documents' AND indexname = 'idx_rag_docs_embedding' ); ")" @@ -225,35 +482,29 @@ ensure_pgvector_store() { has_ann_index="$(printf '%s\n' "$result" | sed -n '4p')" if [[ "$has_table" != "t" ]]; then - echo "Error: local pgvector RAG store was not initialized (rag_documents table missing)." >&2 + error "Local pgvector RAG store was not initialized (rag_documents table missing)." exit 1 fi if [[ "$has_vector" != "t" ]]; then - echo "Error: VECTOR_STORE_TYPE=pgvector but the PostgreSQL 'vector' extension is not installed." >&2 + error "VECTOR_STORE_TYPE=pgvector but the PostgreSQL 'vector' extension is not installed." exit 1 fi if [[ "$embedding_type" != "vector(${expected_dims})" ]]; then - echo "Error: rag_documents.embedding is '$embedding_type' instead of 'vector(${expected_dims})'." >&2 + error "rag_documents.embedding is '$embedding_type' instead of 'vector(${expected_dims})'." exit 1 fi if [[ "$has_ann_index" != "t" ]]; then - echo "Error: local pgvector ANN index idx_rag_docs_embedding is missing." >&2 + error "Local pgvector ANN index idx_rag_docs_embedding is missing." exit 1 fi echo "Verified local pgvector RAG store in the vault database." } -compose() { - DEEPSQL_RUNTIME_ENV_FILE="$ENV_FILE" docker compose \ - --project-name "$PROJECT_NAME" \ - --env-file "$ENV_FILE" \ - -f "$COMPOSE_FILE" \ - "$@" -} +# ── Admin Bootstrap ─────────────────────────────────────────────────────────── bootstrap_admin() { if [[ "${SECURITY_ADMIN_BOOTSTRAP_ENABLED:-false}" != "true" ]]; then @@ -261,43 +512,36 @@ bootstrap_admin() { fi if [[ -z "${ADMIN_BOOTSTRAP_SECRET:-}" || -z "${DEEPSQL_INITIAL_ADMIN_PASSWORD:-}" || -z "${DEEPSQL_INITIAL_ADMIN_EMAIL:-}" ]]; then - echo "Admin bootstrap enabled, but DEEPSQL_INITIAL_ADMIN_EMAIL / DEEPSQL_INITIAL_ADMIN_PASSWORD / ADMIN_BOOTSTRAP_SECRET are not all set. Skipping bootstrap." + warn "Admin bootstrap enabled, but credentials not all set. Skipping bootstrap." return 0 fi - local payload + local payload response payload="$(printf '{\"email\":\"%s\",\"password\":\"%s\"}' \ "${DEEPSQL_INITIAL_ADMIN_EMAIL}" \ "${DEEPSQL_INITIAL_ADMIN_PASSWORD}")" - local response + response="$(printf '%s' "$payload" | compose exec -T \ -e ADMIN_BOOTSTRAP_SECRET="${ADMIN_BOOTSTRAP_SECRET}" \ backend sh -lc \ 'curl -fsS -H "Content-Type: application/json" -H "X-Admin-Bootstrap-Secret: ${ADMIN_BOOTSTRAP_SECRET}" -X POST http://localhost:8080/api/users/admin/reset --data @-' || true)" if [[ "$response" == *"Admin reset successfully"* || "$response" == *"Admin created successfully"* ]]; then - echo "Admin bootstrap complete. Login username: admin" + # Fixed: login is by email, not "username: admin" + echo "Admin bootstrap complete. Login email: ${DEEPSQL_INITIAL_ADMIN_EMAIL}" else - # Previously a warning that the install continued past, so install.sh exited 0 while - # leaving no account to log in with. An installer that cannot create the only user - # has not succeeded, and saying so here beats an opaque 401 from the next command. - echo "Error: admin bootstrap did not return a success message." >&2 + error "Admin bootstrap did not return a success message." echo "$response" >&2 return 1 fi } -# Poll until the credentials just created actually authenticate. -# -# Health being UP is not the same as being able to log in: install.sh flips -# SECURITY_ADMIN_BOOTSTRAP_ENABLED back to false and restarts the backend afterwards, and -# a login issued in the seconds after that restart returns 401. That is what made -# smoke-test.sh -- the very next command install.sh recommends -- fail on a good install. wait_for_login() { local url="http://localhost:${DEEPSQL_BACKEND_PORT:-8080}/api/auth/login" local payload deadline=$((SECONDS + 120)) payload="$(printf '{"email":"%s","password":"%s"}' \ "${DEEPSQL_INITIAL_ADMIN_EMAIL}" "${DEEPSQL_INITIAL_ADMIN_PASSWORD}")" + while (( SECONDS < deadline )); do if curl -fsS -o /dev/null -H 'Content-Type: application/json' \ -X POST "$url" --data "$payload" 2>/dev/null; then @@ -306,11 +550,21 @@ wait_for_login() { fi sleep 5 done - echo "Error: the admin account was created but could not log in within 120s." >&2 + error "The admin account was created but could not log in within 120s." echo "Check 'docker compose logs backend' before running smoke-test.sh." >&2 return 1 } +sed_inplace() { + if [[ "$(uname)" == "Darwin" ]]; then + sed -i '' "$@" + else + sed -i "$@" + fi +} + +# ── Build ───────────────────────────────────────────────────────────────────── + build_application_images() { echo "Building the DeepSQL backend, frontend, and DeepSQL Agent from source..." echo "The first build compiles the Java backend, bundles the frontend, and builds" @@ -319,190 +573,8 @@ build_application_images() { compose build backend frontend deepsql-agent } -require_command docker -require_command curl - -docker compose version >/dev/null 2>&1 || { - echo "Error: docker compose is required." >&2 - exit 1 -} - -if [[ ! -f "$ENV_FILE" ]]; then - cp "$ROOT_DIR/.env.example" "$ENV_FILE" - echo "Created $ENV_FILE from .env.example. Fill in the required values and rerun this script." - exit 1 -fi - -# shellcheck disable=SC1090 -set -a -source "$ENV_FILE" -set +a - -# Auto-generate security secrets if still placeholders -generate_secret SECURITY_JWT_SECRET "openssl rand -base64 64 | tr -d '\n'" -generate_secret ENCRYPTION_KEY "openssl rand -base64 32 | tr -d '\n'" -generate_secret DB_PASSWORD "openssl rand -base64 16 | tr -d '\n'" -generate_secret DEEPSQL_VALKEY_PASSWORD "openssl rand -base64 24 | tr -d '\n'" -generate_secret ADMIN_BOOTSTRAP_SECRET "openssl rand -base64 32 | tr -d '\n'" -generate_secret AGENT_PROVISION_SECRET "openssl rand -base64 32 | tr -d '\n'" - -# Prompt for the chat LLM key if still a placeholder. DeepSQL brings no model -# credentials of its own, and AZURE_OPENAI_* no longer configures chat β€” chat is -# resolved by LlmConfigResolver from DEEPSQL_CHAT_*. -prompt_secret_env_value DEEPSQL_CHAT_API_KEY "LLM API key for chat (e.g. an OpenAI sk-... key)" -# Only when an embedding provider is actually selected β€” otherwise the operator has -# opted into keyword-only retrieval and should not be forced to supply a key. -if [[ -n "${DEEPSQL_EMBEDDING_PROVIDER:-}" ]]; then - prompt_secret_env_value DEEPSQL_EMBEDDING_API_KEY "LLM API key for embeddings (may be the same key)" -fi -prompt_env_value DEEPSQL_INITIAL_ADMIN_EMAIL "Initial admin email" -prompt_secret_env_value DEEPSQL_INITIAL_ADMIN_PASSWORD "Initial admin password" - -# Optional β€” labels this install for analytics + support. If blank the -# backend will derive from the admin email domain on first boot. Either -# can be overridden later by editing this value in $ENV_FILE and restarting. -prompt_optional_env_value DEEPSQL_COMPANY_NAME "Company / organization name (optional, press Enter to skip)" - -: "${SPRING_PROFILES_ACTIVE:=prod}" -: "${DEEPSQL_FRONTEND_PORT:=3000}" -: "${DEEPSQL_BACKEND_PORT:=8080}" -: "${DEEPSQL_POSTGRES_PORT:=5432}" -: "${DEEPSQL_VALKEY_PORT:=6379}" -: "${CORS_ALLOWED_ORIGINS:=http://localhost:${DEEPSQL_FRONTEND_PORT}}" - -if [[ "${VECTOR_STORE_TYPE:-pgvector}" == "pgvector" && -z "${SPRING_AUTOCONFIGURE_EXCLUDE:-}" ]]; then - SPRING_AUTOCONFIGURE_EXCLUDE="org.springframework.ai.vectorstore.azure.autoconfigure.AzureVectorStoreAutoConfiguration" -fi +# ── CLI Setup ───────────────────────────────────────────────────────────────── -export SPRING_PROFILES_ACTIVE -export DEEPSQL_FRONTEND_PORT -export DEEPSQL_BACKEND_PORT -export DEEPSQL_POSTGRES_PORT -export DEEPSQL_VALKEY_PORT -export CORS_ALLOWED_ORIGINS -export SPRING_AUTOCONFIGURE_EXCLUDE -export SECURITY_ADMIN_BOOTSTRAP_ENABLED=true - -sed_inplace "s|^SECURITY_ADMIN_BOOTSTRAP_ENABLED=.*|SECURITY_ADMIN_BOOTSTRAP_ENABLED=true|" "$ENV_FILE" - -require_env_value SECURITY_JWT_SECRET -require_env_value ENCRYPTION_KEY -require_env_value ENCRYPTION_KEY_ID -require_env_value DB_PASSWORD -require_env_value DEEPSQL_VALKEY_PASSWORD -# Chat is resolved by LlmConfigResolver from DEEPSQL_CHAT_*. AZURE_OPENAI_KEY / -# _ENDPOINT / _CHAT_DEPLOYMENT used to be required here; they no longer configure chat. -# _CHAT_DEPLOYMENT is read by nothing at all, and _KEY/_ENDPOINT now feed only the -# optional /api/llm/v1 gateway used by the DeepSQL CLI agent β€” so requiring them -# rejected a perfectly good plain-OpenAI install. -# -# PROVIDER and ENDPOINT are required alongside the key: the resolver ignores every -# other DEEPSQL_CHAT_* value unless PROVIDER is set, and OpenAiCompatibleChatProvider -# reads the endpoint with an empty-string fallback rather than a working default. -require_env_value DEEPSQL_CHAT_PROVIDER -require_env_value DEEPSQL_CHAT_API_KEY -require_env_value DEEPSQL_CHAT_ENDPOINT - -# Embeddings are NOT configured by AZURE_OPENAI_EMBEDDING_DEPLOYMENT, which this script -# used to require. LlmConfigResolver.resolveEmbedding() reads DEEPSQL_EMBEDDING_*, and -# nothing reads that Azure variable any more β€” so requiring it passed the install while -# validating nothing real, and the brain-init diagnostic then pointed the operator back -# at it. -# -# Absence is a degraded mode, not a hard error: the app runs with keyword-only retrieval. -# Do not point the operator at the onboarding wizard here β€” it writes a different, older -# set of config keys that LlmConfigResolver does not read, so it cannot configure this. -if [[ -n "${DEEPSQL_EMBEDDING_PROVIDER:-}" ]]; then - require_env_value DEEPSQL_EMBEDDING_PROVIDER - # Only the key is required: the provider defaults the model (text-embedding-3-large) and - # the endpoint (api.openai.com). Requiring those too would reject a valid plain-OpenAI - # setup that relies on the defaults. - require_env_value DEEPSQL_EMBEDDING_API_KEY -else - echo "Note: DEEPSQL_EMBEDDING_PROVIDER is not set, so no embedding provider is configured." - echo " RAG retrieval stays keyword-only until one is." - echo " To configure it, set DEEPSQL_EMBEDDING_PROVIDER and DEEPSQL_EMBEDDING_API_KEY" - echo " (optionally _MODEL and _ENDPOINT) in $ENV_FILE and re-run this script." -fi - -if [[ "${VECTOR_STORE_TYPE:-pgvector}" == "azure" || "${AZURE_SEARCH_ENABLED:-false}" == "true" ]]; then - require_env_value AZURE_SEARCH_ENDPOINT - require_env_value AZURE_SEARCH_API_KEY - require_env_value AZURE_SEARCH_INDEX_NAME -fi - -echo "Starting DeepSQL self-hosted stack with project '$PROJECT_NAME'..." -build_application_images -compose up -d - -ensure_scheduler_table -ensure_pg_stat_statements -wait_for_http "http://localhost:${DEEPSQL_BACKEND_PORT}/api/actuator/health" "Backend" -wait_for_http "http://localhost:${DEEPSQL_FRONTEND_PORT}" "Frontend" -ensure_pgvector_store - -bootstrap_admin - -sed_inplace "s|^SECURITY_ADMIN_BOOTSTRAP_ENABLED=.*|SECURITY_ADMIN_BOOTSTRAP_ENABLED=false|" "$ENV_FILE" -export SECURITY_ADMIN_BOOTSTRAP_ENABLED=false -compose up -d backend >/dev/null -wait_for_http "http://localhost:${DEEPSQL_BACKEND_PORT}/api/actuator/health" "Backend" - -# The restart above is why this exists: health returns UP before logins are served, so -# without it the installer declares success on a stack that rejects the credentials it -# just printed. -wait_for_login - -echo - -echo "DeepSQL self-hosted stack is ready." -echo "Frontend: http://localhost:${DEEPSQL_FRONTEND_PORT}" -echo "Backend: http://localhost:${DEEPSQL_BACKEND_PORT}/api" -echo "Agent: http://localhost:${DEEPSQL_AGENT_PORT:-8787} (DeepSQL Agent)" -echo "Project: $PROJECT_NAME" -echo "Images: built from source in this checkout" -echo " (backend/Dockerfile, ./Dockerfile, agent/Dockerfile)." -echo " After pulling new code, re-run this script to rebuild." -echo - -# Wait for the DeepSQL Agent container (Agent tab + AI dashboards). -# Host-side setup-agent.sh is only for native (non-Compose) development. -if wait_for_http "http://localhost:${DEEPSQL_AGENT_PROVISIONER_PORT:-8788}/health" "DeepSQL Agent" 60 2; then - echo "DeepSQL Agent is healthy." -else - echo "Warning: DeepSQL Agent did not become healthy in time." >&2 - echo " The core UI still works. Check: docker compose logs deepsql-agent" >&2 -fi -echo - -# Optional host-side agent for native (non-Compose) development only. -# Compose already runs deepsql-agent; skip unless DEEPSQL_HOST_AGENT_SETUP=1. -if [[ "${DEEPSQL_HOST_AGENT_SETUP:-0}" == "1" ]]; then - if [[ -x "$SCRIPT_DIR/setup-agent.sh" ]]; then - echo "Starting host-side DeepSQL Agent (DEEPSQL_HOST_AGENT_SETUP=1)…" - if "$SCRIPT_DIR/setup-agent.sh"; then - echo "Host agent setup complete." - else - echo "Warning: host agent setup failed." >&2 - fi - echo - fi -fi - -# ── DeepSQL CLI (@deepsql/mcp) ─────────────────────────────────────────────── -# Nothing in this repo installed, updated, or logged in the CLI, so a reader -# who followed the README end to end finished with a running stack and no -# `deepsql` command at all β€” and anyone who installed it once drifted silently -# (a machine here sat on 0.16.0 while npm was on 0.26.0). The CLI is an -# agent-facing surface, so a stale one misreports which tools and subcommands -# exist. -# -# Install and log in automatically when npm is available. `npm i -g` is tried -# first without privilege escalation, then retried once with `sudo -n` (never -# an interactive `sudo` β€” a password prompt buried in an otherwise unattended -# installer is exactly the kind of silent hang this script avoids elsewhere). -# Every step here is non-fatal: install or login failure only prints the -# manual command and falls through, it never aborts the installer. install_deepsql_cli() { if npm i -g @deepsql/mcp >/dev/null 2>&1; then return 0 @@ -534,16 +606,13 @@ setup_deepsql_cli() { else echo "DeepSQL CLI: install failed (npm i -g @deepsql/mcp may need elevated" echo " permissions on this system). Install it yourself, then:" - echo " deepsql login --url http://localhost:${DEEPSQL_BACKEND_PORT}" + echo " deepsql login --url http://localhost:${DEEPSQL_BACKEND_PORT:-8080}" echo return 0 fi else - # `npm view` reaches the network; never let it stall or fail the install. latest="$(npm view @deepsql/mcp version 2>/dev/null | tr -d '[:space:]' || true)" if [[ -z "$latest" ]]; then - # Don't claim "up to date" on a check that never completed β€” that is the - # same false-green that let a stale CLI sit unnoticed in the first place. echo "DeepSQL CLI: ${installed} installed (could not reach npm to check for updates)." elif [[ "$installed" != "$latest" ]]; then echo "DeepSQL CLI: ${installed} installed, ${latest} available." @@ -559,59 +628,335 @@ setup_deepsql_cli() { fi if [[ -z "${DEEPSQL_INITIAL_ADMIN_EMAIL:-}" || -z "${DEEPSQL_INITIAL_ADMIN_PASSWORD:-}" ]]; then - echo " Point it at this stack: deepsql login --url http://localhost:${DEEPSQL_BACKEND_PORT}" + echo " Point it at this stack: deepsql login --url http://localhost:${DEEPSQL_BACKEND_PORT:-8080}" echo return 0 fi - # Skip login if a token already exists for this exact stack β€” install.sh is - # meant to be re-run (upgrades, credential rotation), and login mints a new - # long-lived token every time, so re-running it would otherwise pile up - # tokens the operator never asked for under `deepsql whoami`. - if deepsql whoami --url "http://localhost:${DEEPSQL_BACKEND_PORT}" >/dev/null 2>&1; then + if deepsql whoami --url "http://localhost:${DEEPSQL_BACKEND_PORT:-8080}" >/dev/null 2>&1; then echo "DeepSQL CLI: already logged in as ${DEEPSQL_INITIAL_ADMIN_EMAIL}." else echo "Logging in the DeepSQL CLI as ${DEEPSQL_INITIAL_ADMIN_EMAIL}…" if printf '%s' "${DEEPSQL_INITIAL_ADMIN_PASSWORD}" | deepsql login \ - --url "http://localhost:${DEEPSQL_BACKEND_PORT}" --password \ - --email "${DEEPSQL_INITIAL_ADMIN_EMAIL}" --password-stdin --label install; then + --url "http://localhost:${DEEPSQL_BACKEND_PORT:-8080}" --password \ + --email "${DEEPSQL_INITIAL_ADMIN_EMAIL}" --password-stdin --label install 2>/dev/null; then : else echo "DeepSQL CLI: login failed. Run manually:" - echo " deepsql login --url http://localhost:${DEEPSQL_BACKEND_PORT}" + echo " deepsql login --url http://localhost:${DEEPSQL_BACKEND_PORT:-8080}" fi fi echo } -setup_deepsql_cli +# ── Demo Seeding ────────────────────────────────────────────────────────────── + +run_demo_seed() { + if [[ "$SEED_DEMO" -eq 0 ]]; then + echo "Demo data seeding skipped (--no-seed-demo)." + echo " Run ./scripts/self-host/seed-demo-data.sh for a ready-to-explore demo database." + echo + return 0 + fi -# ── Demo Data Seeding ───────────────────────────────────────────────────────── -# Optional: seed a demo database with sample e-commerce data, users, saved queries, -# and performance recommendations. Gives new users an end-to-end view of all features. -# Enable with DEEPSQL_SEED_DEMO_DATA=1 in .env or environment. -if [[ "${DEEPSQL_SEED_DEMO_DATA:-0}" == "1" ]]; then if [[ -x "$SCRIPT_DIR/seed-demo-data.sh" ]]; then - echo "Seeding demo data (DEEPSQL_SEED_DEMO_DATA=1)…" + echo "Seeding demo data..." + # Seed failure is a warning, not an install failure if "$SCRIPT_DIR/seed-demo-data.sh"; then echo "Demo data seeding complete." else - echo "Warning: demo data seeding failed. The stack still works, but the demo" >&2 - echo " database and sample data were not created. Run manually:" >&2 - echo " ./scripts/self-host/seed-demo-data.sh" >&2 + warn "Demo data seeding had issues. The stack still works." + echo " Run manually: ./scripts/self-host/seed-demo-data.sh" fi echo + else + warn "seed-demo-data.sh not found. Skipping demo seeding." fi -else - echo "Demo data seeding skipped (set DEEPSQL_SEED_DEMO_DATA=1 to enable)." - echo " Run ./scripts/self-host/seed-demo-data.sh for a ready-to-explore demo database." +} + +# ── Print Final Summary ─────────────────────────────────────────────────────── + +print_summary() { + local frontend_port="${DEEPSQL_FRONTEND_PORT:-3000}" + local backend_port="${DEEPSQL_BACKEND_PORT:-8080}" + local has_llm_key=0 + [[ -n "${DEEPSQL_CHAT_API_KEY:-}" ]] && ! is_placeholder "${DEEPSQL_CHAT_API_KEY:-}" && has_llm_key=1 + echo -fi + echo "${BOLD}═══════════════════════════════════════════════════════════════════════════${NC}" + echo "${GREEN}${BOLD} DeepSQL is running!${NC}" + echo "${BOLD}═══════════════════════════════════════════════════════════════════════════${NC}" + echo + echo " ${BOLD}Login URL:${NC} http://localhost:${frontend_port}" + echo " ${BOLD}Login email:${NC} ${DEEPSQL_INITIAL_ADMIN_EMAIL}" + echo " ${BOLD}Password:${NC} stored in ${ENV_FILE}" + echo " ${BOLD}Health URL:${NC} http://localhost:${backend_port}/api/actuator/health" + echo + + if [[ "$has_llm_key" -eq 0 ]]; then + echo "${YELLOW} Note: No LLM key configured. Chat and AI features are disabled.${NC}" + echo " Add the key during onboarding in the web UI, or re-run the installer with:" + echo " DEEPSQL_LLM_API_KEY= ./scripts/self-host/install.sh" + echo + fi + + echo "Project: $PROJECT_NAME" + echo "Images: built from source in this checkout" + echo + echo "Useful commands:" + echo " ./scripts/self-host/status.sh" + echo " ./scripts/self-host/smoke-test.sh" + echo " ./scripts/self-host/seed-demo-data.sh # Seed demo e-commerce database" + echo " docker compose logs -f backend # Backend logs" + echo " ./scripts/self-host/uninstall.sh" +} + +# ═══════════════════════════════════════════════════════════════════════════════ +# MAIN +# ═══════════════════════════════════════════════════════════════════════════════ + +main() { + echo + echo "${BOLD}═══════════════════════════════════════════════════════════════════════════${NC}" + echo "${BOLD} DeepSQL Self-Host Installer${NC}" + echo "${BOLD}═══════════════════════════════════════════════════════════════════════════${NC}" + echo + + check_prerequisites + + # ── Create .env if needed ─────────────────────────────────────────────────── + if [[ ! -f "$ENV_FILE" ]]; then + if [[ -f "$ROOT_DIR/.env.example" ]]; then + cp "$ROOT_DIR/.env.example" "$ENV_FILE" + info "Created $ENV_FILE from .env.example" + else + error ".env.example not found in checkout." + exit 1 + fi + fi + + # ── Env var aliasing ───────────────────────────────────────────────────────── + # DEEPSQL_LLM_* is the primary documented name; DEEPSQL_CHAT_* is the alias. + # The backend uses DEEPSQL_CHAT_*, so we map LLM->CHAT here. + if [[ -n "${DEEPSQL_LLM_API_KEY:-}" ]]; then + export DEEPSQL_CHAT_API_KEY="${DEEPSQL_CHAT_API_KEY:-$DEEPSQL_LLM_API_KEY}" + fi + if [[ -n "${DEEPSQL_LLM_PROVIDER:-}" ]]; then + export DEEPSQL_CHAT_PROVIDER="${DEEPSQL_CHAT_PROVIDER:-$DEEPSQL_LLM_PROVIDER}" + fi + if [[ -n "${DEEPSQL_LLM_BASE_URL:-}" ]]; then + export DEEPSQL_CHAT_ENDPOINT="${DEEPSQL_CHAT_ENDPOINT:-$DEEPSQL_LLM_BASE_URL}" + fi + if [[ -n "${DEEPSQL_LLM_MODEL:-}" ]]; then + export DEEPSQL_CHAT_MODEL="${DEEPSQL_CHAT_MODEL:-$DEEPSQL_LLM_MODEL}" + fi + + # ── Load .env but let env vars take precedence ────────────────────────────── + # Store current env vars that should override .env + declare -A override_vars + for var in DEEPSQL_CHAT_API_KEY DEEPSQL_CHAT_PROVIDER DEEPSQL_CHAT_ENDPOINT DEEPSQL_CHAT_MODEL \ + DEEPSQL_LLM_API_KEY DEEPSQL_LLM_PROVIDER DEEPSQL_LLM_BASE_URL DEEPSQL_LLM_MODEL \ + DEEPSQL_EMBEDDING_PROVIDER DEEPSQL_EMBEDDING_API_KEY DEEPSQL_EMBEDDING_ENDPOINT DEEPSQL_EMBEDDING_MODEL \ + DEEPSQL_INITIAL_ADMIN_EMAIL DEEPSQL_INITIAL_ADMIN_PASSWORD \ + DEEPSQL_FRONTEND_PORT DEEPSQL_BACKEND_PORT DEEPSQL_POSTGRES_PORT DEEPSQL_VALKEY_PORT \ + SECURITY_JWT_SECRET ENCRYPTION_KEY DB_PASSWORD DEEPSQL_VALKEY_PASSWORD \ + ADMIN_BOOTSTRAP_SECRET AGENT_PROVISION_SECRET DEEPSQL_COMPANY_NAME; do + if [[ -n "${!var:-}" ]]; then + override_vars[$var]="${!var}" + fi + done + + # Source .env (path determined at runtime) + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a + + # Restore overrides (env vars take precedence over .env placeholders) + for var in "${!override_vars[@]}"; do + export "$var=${override_vars[$var]}" + done + + # Re-apply LLM->CHAT aliasing after sourcing .env (in case .env had DEEPSQL_LLM_*) + if [[ -n "${DEEPSQL_LLM_API_KEY:-}" ]] && [[ -z "${DEEPSQL_CHAT_API_KEY:-}" || "${DEEPSQL_CHAT_API_KEY}" == replace-with-* ]]; then + export DEEPSQL_CHAT_API_KEY="$DEEPSQL_LLM_API_KEY" + fi + if [[ -n "${DEEPSQL_LLM_PROVIDER:-}" ]] && [[ -z "${DEEPSQL_CHAT_PROVIDER:-}" || "${DEEPSQL_CHAT_PROVIDER}" == "openai" ]]; then + export DEEPSQL_CHAT_PROVIDER="$DEEPSQL_LLM_PROVIDER" + fi + if [[ -n "${DEEPSQL_LLM_BASE_URL:-}" ]] && [[ -z "${DEEPSQL_CHAT_ENDPOINT:-}" || "${DEEPSQL_CHAT_ENDPOINT}" == https://api.openai.com* ]]; then + export DEEPSQL_CHAT_ENDPOINT="$DEEPSQL_LLM_BASE_URL" + fi + if [[ -n "${DEEPSQL_LLM_MODEL:-}" ]] && [[ -z "${DEEPSQL_CHAT_MODEL:-}" || "${DEEPSQL_CHAT_MODEL}" == "gpt-4o" ]]; then + export DEEPSQL_CHAT_MODEL="$DEEPSQL_LLM_MODEL" + fi + + # ── Check for existing volumes ────────────────────────────────────────────── + check_existing_volumes + + # ── Auto-generate security secrets if still placeholders ──────────────────── + generate_secret SECURITY_JWT_SECRET "openssl rand -base64 64 | tr -d '\n'" + generate_secret ENCRYPTION_KEY "openssl rand -base64 32 | tr -d '\n'" + generate_secret DB_PASSWORD "openssl rand -base64 16 | tr -d '\n'" + generate_secret DEEPSQL_VALKEY_PASSWORD "openssl rand -base64 24 | tr -d '\n'" + generate_secret ADMIN_BOOTSTRAP_SECRET "openssl rand -base64 32 | tr -d '\n'" + generate_secret AGENT_PROVISION_SECRET "openssl rand -base64 32 | tr -d '\n'" + + # ── LLM Configuration (optional - keyless start allowed) ──────────────────── + # Set defaults for provider/endpoint/model if key is provided + if [[ -n "${DEEPSQL_CHAT_API_KEY:-}" ]] && ! is_placeholder "${DEEPSQL_CHAT_API_KEY:-}"; then + # Key is set, ensure provider and endpoint have defaults + if is_placeholder "${DEEPSQL_CHAT_PROVIDER:-}"; then + write_env_value DEEPSQL_CHAT_PROVIDER "openai" + fi + if is_placeholder "${DEEPSQL_CHAT_ENDPOINT:-}"; then + write_env_value DEEPSQL_CHAT_ENDPOINT "https://api.openai.com/v1" + fi + if is_placeholder "${DEEPSQL_CHAT_MODEL:-}"; then + write_env_value DEEPSQL_CHAT_MODEL "gpt-4o" + fi + else + # No key - prompt if interactive, otherwise allow keyless start + if can_prompt; then + echo + echo "LLM API key (e.g., OpenAI sk-... key)." + echo "Press Enter to skip and configure later during onboarding in the web UI." + prompt_value DEEPSQL_CHAT_API_KEY "LLM API key" 0 1 + fi + + if [[ -n "${DEEPSQL_CHAT_API_KEY:-}" ]] && ! is_placeholder "${DEEPSQL_CHAT_API_KEY:-}"; then + # User provided key - set defaults + if is_placeholder "${DEEPSQL_CHAT_PROVIDER:-}"; then + write_env_value DEEPSQL_CHAT_PROVIDER "openai" + fi + if is_placeholder "${DEEPSQL_CHAT_ENDPOINT:-}"; then + write_env_value DEEPSQL_CHAT_ENDPOINT "https://api.openai.com/v1" + fi + if is_placeholder "${DEEPSQL_CHAT_MODEL:-}"; then + write_env_value DEEPSQL_CHAT_MODEL "gpt-4o" + fi + else + # Keyless start - clear placeholders so backend doesn't reject them + if is_placeholder "${DEEPSQL_CHAT_API_KEY:-}"; then + write_env_value DEEPSQL_CHAT_API_KEY "" + fi + if is_placeholder "${DEEPSQL_CHAT_PROVIDER:-}"; then + write_env_value DEEPSQL_CHAT_PROVIDER "" + fi + echo + echo "No LLM key configured. Chat and AI features will be disabled." + echo "Configure your LLM key during onboarding in the web UI after logging in." + # Print machine-readable output when non-interactive OR when no TTY available + if [[ "$NON_INTERACTIVE" -eq 1 ]] || ! can_prompt; then + echo "NEEDS_USER_INPUT: DEEPSQL_LLM_API_KEY (optional, can be set during onboarding at http://localhost:${DEEPSQL_FRONTEND_PORT:-3000})" + fi + fi + fi + + # ── Admin account ─────────────────────────────────────────────────────────── + prompt_value DEEPSQL_INITIAL_ADMIN_EMAIL "Initial admin email" 1 0 || { + if [[ "$NON_INTERACTIVE" -eq 1 ]]; then + error "DEEPSQL_INITIAL_ADMIN_EMAIL is required for non-interactive install." + echo "Set it via environment variable: DEEPSQL_INITIAL_ADMIN_EMAIL=admin@example.com" + fi + exit 1 + } + + # Generate password if not provided + if is_placeholder "${DEEPSQL_INITIAL_ADMIN_PASSWORD:-}"; then + local gen_password + gen_password="$(openssl rand -base64 16 | tr -d '\n')" + write_env_value DEEPSQL_INITIAL_ADMIN_PASSWORD "$gen_password" + echo "Auto-generated admin password (saved to .env)." + fi + + # Optional company name + if can_prompt && is_placeholder "${DEEPSQL_COMPANY_NAME:-}"; then + prompt_value DEEPSQL_COMPANY_NAME "Company / organization name (optional, press Enter to skip)" 0 0 + fi + + # ── Export required variables ─────────────────────────────────────────────── + : "${SPRING_PROFILES_ACTIVE:=prod}" + : "${DEEPSQL_FRONTEND_PORT:=3000}" + : "${DEEPSQL_BACKEND_PORT:=8080}" + : "${DEEPSQL_POSTGRES_PORT:=5432}" + : "${DEEPSQL_VALKEY_PORT:=6379}" + : "${CORS_ALLOWED_ORIGINS:=http://localhost:${DEEPSQL_FRONTEND_PORT}}" + + if [[ "${VECTOR_STORE_TYPE:-pgvector}" == "pgvector" && -z "${SPRING_AUTOCONFIGURE_EXCLUDE:-}" ]]; then + SPRING_AUTOCONFIGURE_EXCLUDE="org.springframework.ai.vectorstore.azure.autoconfigure.AzureVectorStoreAutoConfiguration" + fi + + export SPRING_PROFILES_ACTIVE DEEPSQL_FRONTEND_PORT DEEPSQL_BACKEND_PORT + export DEEPSQL_POSTGRES_PORT DEEPSQL_VALKEY_PORT CORS_ALLOWED_ORIGINS + export SPRING_AUTOCONFIGURE_EXCLUDE + export SECURITY_ADMIN_BOOTSTRAP_ENABLED=true + + sed_inplace "s|^SECURITY_ADMIN_BOOTSTRAP_ENABLED=.*|SECURITY_ADMIN_BOOTSTRAP_ENABLED=true|" "$ENV_FILE" + + # ── Validate required secrets ─────────────────────────────────────────────── + local missing_secrets=0 + for var in SECURITY_JWT_SECRET ENCRYPTION_KEY ENCRYPTION_KEY_ID DB_PASSWORD DEEPSQL_VALKEY_PASSWORD; do + if is_placeholder "${!var:-}"; then + error "'$var' must be set." + missing_secrets=1 + fi + done + [[ "$missing_secrets" -eq 1 ]] && exit 1 + + # ── Build and start ───────────────────────────────────────────────────────── + echo + info "Starting DeepSQL self-hosted stack with project '$PROJECT_NAME'..." + build_application_images + compose up -d + + ensure_scheduler_table + ensure_pg_stat_statements + wait_for_http "http://localhost:${DEEPSQL_BACKEND_PORT}/api/actuator/health" "Backend" + wait_for_http "http://localhost:${DEEPSQL_FRONTEND_PORT}" "Frontend" + ensure_pgvector_store + + bootstrap_admin + + # Disable bootstrap and restart backend + sed_inplace "s|^SECURITY_ADMIN_BOOTSTRAP_ENABLED=.*|SECURITY_ADMIN_BOOTSTRAP_ENABLED=false|" "$ENV_FILE" + export SECURITY_ADMIN_BOOTSTRAP_ENABLED=false + compose up -d backend >/dev/null + wait_for_http "http://localhost:${DEEPSQL_BACKEND_PORT}/api/actuator/health" "Backend" + wait_for_login + + echo + + # Wait for agent + if wait_for_http "http://localhost:${DEEPSQL_AGENT_PROVISIONER_PORT:-8788}/health" "DeepSQL Agent" 60 2; then + echo "DeepSQL Agent is healthy." + else + warn "DeepSQL Agent did not become healthy in time." + echo " The core UI still works. Check: docker compose logs deepsql-agent" + fi + echo + + # Host-side agent (only for native development) + if [[ "${DEEPSQL_HOST_AGENT_SETUP:-0}" == "1" ]]; then + if [[ -x "$SCRIPT_DIR/setup-agent.sh" ]]; then + echo "Starting host-side DeepSQL Agent (DEEPSQL_HOST_AGENT_SETUP=1)…" + if "$SCRIPT_DIR/setup-agent.sh"; then + echo "Host agent setup complete." + else + warn "Host agent setup failed." + fi + echo + fi + fi + + # CLI setup + setup_deepsql_cli + + # Demo seeding + run_demo_seed + + # Final summary + print_summary +} -echo "Useful commands:" -echo " ./scripts/self-host/status.sh" -echo " ./scripts/self-host/smoke-test.sh" -echo " ./scripts/self-host/seed-demo-data.sh # Seed demo e-commerce database" -echo " python3 scripts/self-host/e2e-agent-check.py # live Agent+dashboard turn" -echo " docker compose logs deepsql-agent # DeepSQL Agent logs" -echo " ./scripts/self-host/uninstall.sh" +main diff --git a/scripts/self-host/remote-install.sh b/scripts/self-host/remote-install.sh index f826187..a87ef66 100755 --- a/scripts/self-host/remote-install.sh +++ b/scripts/self-host/remote-install.sh @@ -6,6 +6,13 @@ # One-liner install: # curl -fsSL https://deepsql.ai/install.sh | bash # +# This script clones the repository and runs the full install. Pass environment +# variables for LLM credentials and admin account; everything else is generated +# or prompted. +# +# For AI agents (non-interactive): +# DEEPSQL_LLM_API_KEY=sk-... curl -fsSL https://deepsql.ai/install.sh | bash +# # Fallback (raw GitHub URL): # curl -fsSL https://raw.githubusercontent.com/DeepSQLAI/deepsql/main/scripts/self-host/remote-install.sh | bash # @@ -28,7 +35,10 @@ set -euo pipefail DEEPSQL_REPO="DeepSQLAI/deepsql" DEEPSQL_HOME="${DEEPSQL_HOME:-$HOME/deepsql}" -DEEPSQL_BRANCH="${DEEPSQL_BRANCH:-}" # empty = auto-detect latest release tag +DEEPSQL_REF="${DEEPSQL_REF:-}" # empty = auto-detect latest release tag + +# Script version (updated with releases) +REMOTE_INSTALLER_VERSION="1.4.0" # Colors (disabled if not a terminal) if [[ -t 1 ]]; then @@ -42,9 +52,9 @@ else RED='' GREEN='' YELLOW='' BLUE='' BOLD='' NC='' fi -info() { printf "${BLUE}==>${NC} %s\n" "$*"; } -warn() { printf "${YELLOW}Warning:${NC} %s\n" "$*" >&2; } -error() { printf "${RED}Error:${NC} %s\n" "$*" >&2; } +info() { printf "${BLUE}==>${NC} %s\n" "$*"; } +warn() { printf "${YELLOW}Warning:${NC} %s\n" "$*" >&2; } +error() { printf "${RED}Error:${NC} %s\n" "$*" >&2; } success() { printf "${GREEN}βœ“${NC} %s\n" "$*"; } # ── OS / Architecture Detection ─────────────────────────────────────────────── @@ -107,6 +117,34 @@ version_ge() { [[ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | head -1)" == "$2" ]] } +check_docker_permission() { + if ! docker info >/dev/null 2>&1; then + local docker_err + docker_err="$(docker info 2>&1 || true)" + + if echo "$docker_err" | grep -qi "permission denied\|connect: permission denied\|Got permission denied"; then + error "Docker permission denied." + echo + echo "Your user is not in the docker group. Fix with:" + echo + echo " ${BOLD}sudo usermod -aG docker \$USER${NC}" + echo " ${BOLD}newgrp docker${NC} # or log out and back in" + echo + echo "Then re-run this installer:" + echo " curl -fsSL https://deepsql.ai/install.sh | bash" + exit 1 + elif echo "$docker_err" | grep -qi "Is the docker daemon running\|Cannot connect"; then + error "Docker daemon is not running." + echo + echo "Start Docker with:" + echo " sudo systemctl start docker" + echo + echo "Then re-run this installer." + exit 1 + fi + fi +} + check_docker() { info "Checking Docker..." @@ -129,17 +167,7 @@ check_docker() { exit 1 fi - if ! docker info >/dev/null 2>&1; then - error "Docker daemon is not running or you lack permission." - echo - echo "If Docker is installed but not running:" - echo " sudo systemctl start docker" - echo - echo "If you need permission:" - echo " sudo usermod -aG docker \$USER" - echo " # Then log out and back in" - exit 1 - fi + check_docker_permission success "Docker is running" # Check Compose v2 @@ -159,7 +187,7 @@ check_docker() { fi success "Docker Compose $compose_version" - # Check buildx (required for multi-stage builds) + # Check buildx local buildx_version buildx_version="$(docker buildx version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)" if [[ -z "$buildx_version" ]]; then @@ -183,33 +211,59 @@ check_docker() { success "Docker buildx $buildx_version" } -# ── Release Tag Detection ───────────────────────────────────────────────────── +# ── Release Tag Detection with Retry ────────────────────────────────────────── get_latest_release_tag() { - # Query GitHub API for tags, filter to product tags (v[0-9]*), exclude desktop-v* tags - local tags - tags="$(curl -fsSL "https://api.github.com/repos/${DEEPSQL_REPO}/tags?per_page=50" 2>/dev/null || true)" + local max_retries=3 + local retry_delay=2 + local attempt - if [[ -z "$tags" ]]; then - return 1 - fi + for ((attempt=1; attempt<=max_retries; attempt++)); do + local tags + tags="$(curl -fsSL --retry 2 "https://api.github.com/repos/${DEEPSQL_REPO}/tags?per_page=50" 2>/dev/null || true)" + + if [[ -n "$tags" ]]; then + # Extract tag names, filter to product releases only: + # - Must start with v followed by a digit (v1.0.0, v2.3.4, etc.) + # - Excludes desktop-v*, agent-v*, or any other prefixed tags + local latest + latest="$(echo "$tags" \ + | grep -o '"name": *"[^"]*"' \ + | cut -d'"' -f4 \ + | grep -E '^v[0-9]' \ + | grep -v '^desktop-' \ + | sort -V -r \ + | head -1 || true)" + + if [[ -n "$latest" ]]; then + echo "$latest" + return 0 + fi + fi + + if [[ "$attempt" -lt "$max_retries" ]]; then + sleep "$retry_delay" + retry_delay=$((retry_delay * 2)) + fi + done - # Extract tag names, filter to product releases only: - # - Must start with v followed by a digit (v1.0.0, v2.3.4, etc.) - # - Excludes desktop-v*, agent-v*, or any other prefixed tags - local latest - latest="$(echo "$tags" \ - | grep -o '"name": *"[^"]*"' \ - | cut -d'"' -f4 \ - | grep -E '^v[0-9]' \ - | grep -v '^desktop-' \ - | head -1 || true)" + return 1 +} + +# Fallback: use git ls-remote (not rate-limited like API) +get_latest_release_tag_git() { + local tags + tags="$(git ls-remote --tags "https://github.com/${DEEPSQL_REPO}.git" 2>/dev/null | \ + grep -oE 'refs/tags/v[0-9][^{]*$' | \ + sed 's|refs/tags/||' | \ + grep -v '^desktop-' | \ + sort -V -r | \ + head -1 || true)" - if [[ -n "$latest" ]]; then - echo "$latest" + if [[ -n "$tags" ]]; then + echo "$tags" return 0 fi - return 1 } @@ -218,20 +272,20 @@ get_latest_release_tag() { clone_or_update() { local target_ref="$1" + # When piped, git commands can consume stdin. Redirect from /dev/null. if [[ -d "$DEEPSQL_HOME/.git" ]]; then info "Updating existing checkout at $DEEPSQL_HOME..." cd "$DEEPSQL_HOME" # Fetch latest (include tags) - if ! git fetch --tags origin 2>/dev/null; then + if ! git fetch --tags origin /dev/null; then warn "Failed to fetch updates. Continuing with existing checkout." fi if [[ -n "$target_ref" ]]; then info "Checking out $target_ref..." - git checkout "$target_ref" 2>/dev/null || git checkout -b "$target_ref" "origin/$target_ref" 2>/dev/null || { - # If it's a tag, just checkout directly - git checkout "$target_ref" 2>/dev/null || { + git checkout "$target_ref" /dev/null || git checkout -b "$target_ref" "origin/$target_ref" /dev/null || { + git checkout "$target_ref" /dev/null || { warn "Could not checkout $target_ref. Staying on current branch." } } @@ -249,7 +303,7 @@ clone_or_update() { clone_args+=(--branch "$target_ref") fi - if ! git clone "${clone_args[@]}" "https://github.com/${DEEPSQL_REPO}.git" "$DEEPSQL_HOME"; then + if ! git clone "${clone_args[@]}" "https://github.com/${DEEPSQL_REPO}.git" "$DEEPSQL_HOME" Use a specific branch or tag instead of latest release - -Environment variables: - DEEPSQL_HOME Installation directory (default: \$HOME/deepsql) - DEEPSQL_BRANCH Branch or tag to checkout (default: latest release) + --ref Use a specific branch or tag instead of latest release + --non-interactive Never prompt; use env vars or defaults + --seed-demo Seed demo database after install (default) + --no-seed-demo Skip demo database seeding + --fresh Remove existing volumes before install + --project-name NAME Set Compose project name + +Environment variables (override .env placeholders): + DEEPSQL_HOME Installation directory (default: \$HOME/deepsql) + DEEPSQL_REF Branch or tag to checkout (default: latest release) + DEEPSQL_LLM_API_KEY LLM key (optional - can set later in the web UI) + DEEPSQL_LLM_PROVIDER Provider id (default: openai) + DEEPSQL_LLM_BASE_URL API endpoint (default: https://api.openai.com/v1) + DEEPSQL_LLM_MODEL Model name (default: gpt-4o) + DEEPSQL_INITIAL_ADMIN_EMAIL Admin login email + DEEPSQL_INITIAL_ADMIN_PASSWORD Admin password (generated if unset) Examples: - # Interactive setup (edit .env, then run install.sh yourself) + # Interactive install curl -fsSL https://deepsql.ai/install.sh | bash - # Noninteractive (requires .env to be pre-configured or prompts) - curl -fsSL https://deepsql.ai/install.sh | bash -s -- --yes + # With LLM key (non-interactive for AI agents) + DEEPSQL_LLM_API_KEY=sk-... curl -fsSL https://deepsql.ai/install.sh | bash + + # Keyless install (configure LLM later in UI) + DEEPSQL_INITIAL_ADMIN_EMAIL=admin@example.com \\ + curl -fsSL https://deepsql.ai/install.sh | bash # Install a specific version - curl -fsSL https://deepsql.ai/install.sh | bash -s -- --branch v1.3.0 + curl -fsSL https://deepsql.ai/install.sh | bash -s -- --ref v1.3.0 + + # Fresh install (removes existing data) + curl -fsSL https://deepsql.ai/install.sh | bash -s -- --fresh + +For AI agents: + Ask the user for their LLM API key (OpenAI, Azure OpenAI, etc.), then run: + + DEEPSQL_LLM_API_KEY= curl -fsSL https://deepsql.ai/install.sh | bash - # Fallback: raw GitHub URL (if deepsql.ai is unreachable) - curl -fsSL https://raw.githubusercontent.com/DeepSQLAI/deepsql/main/scripts/self-host/remote-install.sh | bash + After install: + Health: curl -fsS http://localhost:8080/api/actuator/health + Login: http://localhost:3000 (credentials in ~/deepsql/.env) EOF } main() { - local auto_install=0 local skip_docker=0 + local install_args=() while [[ $# -gt 0 ]]; do case "$1" in @@ -414,21 +449,37 @@ main() { usage exit 0 ;; - -y|--yes) - auto_install=1 - shift + -V|--version) + echo "DeepSQL Remote Installer $REMOTE_INSTALLER_VERSION" + exit 0 ;; --skip-docker-check) skip_docker=1 shift ;; - --branch) - DEEPSQL_BRANCH="$2" + --ref|--branch) + DEEPSQL_REF="$2" + shift 2 + ;; + --ref=*|--branch=*) + DEEPSQL_REF="${1#*=}" + shift + ;; + --non-interactive|--seed-demo|--no-seed-demo|--fresh) + install_args+=("$1") + shift + ;; + --project-name) + install_args+=("$1" "$2") shift 2 ;; + --project-name=*) + install_args+=("--project-name" "${1#*=}") + shift + ;; *) error "Unknown option: $1" - usage + echo "Run with --help for usage." >&2 exit 1 ;; esac @@ -450,28 +501,38 @@ main() { fi # Determine target ref - local target_ref="$DEEPSQL_BRANCH" + local target_ref="$DEEPSQL_REF" if [[ -z "$target_ref" ]]; then info "Finding latest stable release..." target_ref="$(get_latest_release_tag || true)" + + # Fallback to git ls-remote if API fails + if [[ -z "$target_ref" ]]; then + target_ref="$(get_latest_release_tag_git || true)" + fi + if [[ -z "$target_ref" ]]; then - target_ref="main" - warn "Could not determine latest release. Using default branch: $target_ref" + # Fail loudly instead of silently falling back to main + error "Could not determine latest release." + echo + echo "This may be due to GitHub API rate limits. Try one of:" + echo " 1. Wait a few minutes and try again" + echo " 2. Specify a version explicitly:" + echo " curl -fsSL https://deepsql.ai/install.sh | bash -s -- --ref v1.3.0" + echo " 3. Check available releases: https://github.com/${DEEPSQL_REPO}/releases" + exit 1 else success "Latest release: $target_ref" fi + else + info "Using specified ref: $target_ref" fi clone_or_update "$target_ref" setup_env - if [[ "$auto_install" -eq 1 ]]; then - print_llm_setup - run_install - else - print_llm_setup - print_final_instructions - fi + # Run install with passed-through arguments + run_install "${install_args[@]}" } main "$@" diff --git a/scripts/self-host/smoke-test.sh b/scripts/self-host/smoke-test.sh index 575961d..84cb598 100755 --- a/scripts/self-host/smoke-test.sh +++ b/scripts/self-host/smoke-test.sh @@ -12,8 +12,8 @@ if [[ ! -f "$ENV_FILE" ]]; then exit 1 fi -# shellcheck disable=SC1090 set -a +# shellcheck disable=SC1090 source "$ENV_FILE" set +a diff --git a/scripts/self-host/status.sh b/scripts/self-host/status.sh index 21e18fa..288855c 100755 --- a/scripts/self-host/status.sh +++ b/scripts/self-host/status.sh @@ -16,8 +16,8 @@ compose() { } if [[ -f "$ENV_FILE" ]]; then - # shellcheck disable=SC1090 set -a + # shellcheck disable=SC1090 source "$ENV_FILE" set +a fi